1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
3
4 #include <linux/completion.h>
5 #include <linux/circ_buf.h>
6 #include <linux/list.h>
7
8 #include <soc/qcom/cmd-db.h>
9 #include <soc/qcom/tcs.h>
10
11 #include "a6xx_gmu.h"
12 #include "a6xx_gmu.xml.h"
13 #include "a6xx_gpu.h"
14
15 #define HFI_MSG_ID(val) [val] = #val
16
17 static const char * const a6xx_hfi_msg_id[] = {
18 HFI_MSG_ID(HFI_H2F_MSG_INIT),
19 HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION),
20 HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
21 HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
22 HFI_MSG_ID(HFI_H2F_MSG_TEST),
23 HFI_MSG_ID(HFI_H2F_MSG_START),
24 HFI_MSG_ID(HFI_H2F_FEATURE_CTRL),
25 HFI_MSG_ID(HFI_H2F_MSG_CORE_FW_START),
26 HFI_MSG_ID(HFI_H2F_MSG_TABLE),
27 HFI_MSG_ID(HFI_H2F_MSG_GX_BW_PERF_VOTE),
28 HFI_MSG_ID(HFI_H2F_MSG_PREPARE_SLUMBER),
29 };
30
a6xx_hfi_queue_read(struct a6xx_gmu * gmu,struct a6xx_hfi_queue * queue,u32 * data,u32 dwords)31 static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu,
32 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
33 {
34 struct a6xx_hfi_queue_header *header = queue->header;
35 u32 i, hdr, index = header->read_index;
36
37 if (header->read_index == READ_ONCE(header->write_index)) {
38 header->rx_request = 1;
39 return 0;
40 }
41
42 hdr = queue->data[index];
43
44 queue->history[(queue->history_idx++) % HFI_HISTORY_SZ] = index;
45
46 /*
47 * If we are to assume that the GMU firmware is in fact a rational actor
48 * and is programmed to not send us a larger response than we expect
49 * then we can also assume that if the header size is unexpectedly large
50 * that it is due to memory corruption and/or hardware failure. In this
51 * case the only reasonable course of action is to BUG() to help harden
52 * the failure.
53 */
54
55 BUG_ON(HFI_HEADER_SIZE(hdr) > dwords);
56
57 for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {
58 data[i] = queue->data[index];
59 index = (index + 1) % header->size;
60 }
61
62 if (!gmu->legacy)
63 index = ALIGN(index, 4) % header->size;
64
65 /* Ensure all memory operations are complete before updating the read index */
66 dma_mb();
67
68 WRITE_ONCE(header->read_index, index);
69 return HFI_HEADER_SIZE(hdr);
70 }
71
a6xx_hfi_queue_write(struct a6xx_gmu * gmu,struct a6xx_hfi_queue * queue,u32 * data,u32 dwords)72 static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
73 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
74 {
75 struct a6xx_hfi_queue_header *header = queue->header;
76 u32 i, space, index = header->write_index;
77
78 spin_lock(&queue->lock);
79
80 space = CIRC_SPACE(header->write_index, READ_ONCE(header->read_index),
81 header->size);
82 if (space < dwords) {
83 header->dropped++;
84 spin_unlock(&queue->lock);
85 return -ENOSPC;
86 }
87
88 queue->history[(queue->history_idx++) % HFI_HISTORY_SZ] = index;
89
90 for (i = 0; i < dwords; i++) {
91 queue->data[index] = data[i];
92 index = (index + 1) % header->size;
93 }
94
95 /* Cookify any non used data at the end of the write buffer */
96 if (!gmu->legacy) {
97 for (; index % 4; index = (index + 1) % header->size)
98 queue->data[index] = 0xfafafafa;
99 }
100
101 /* Ensure all memory operations are complete before updating the write index */
102 dma_mb();
103
104 WRITE_ONCE(header->write_index, index);
105 spin_unlock(&queue->lock);
106
107 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);
108 return 0;
109 }
110
a6xx_hfi_wait_for_msg_interrupt(struct a6xx_gmu * gmu,u32 id,u32 seqnum)111 static int a6xx_hfi_wait_for_msg_interrupt(struct a6xx_gmu *gmu, u32 id, u32 seqnum)
112 {
113 int ret;
114 u32 val;
115 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
116
117 do {
118 /* Wait for a response */
119 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
120 val & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 100, 1000000);
121
122 if (!ret)
123 break;
124
125 if (completion_done(&a6xx_gpu->base.fault_coredump_done))
126 break;
127
128 /* We may timeout because the GMU is temporarily wedged from
129 * pending faults from the GPU and we are taking a devcoredump.
130 * Wait until the MMU is resumed and try again.
131 */
132 wait_for_completion(&a6xx_gpu->base.fault_coredump_done);
133 } while (true);
134
135 if (ret) {
136 DRM_DEV_ERROR(gmu->dev,
137 "Message %s id %d timed out waiting for response\n",
138 a6xx_hfi_msg_id[id], seqnum);
139 return -ETIMEDOUT;
140 }
141
142 /* Clear the interrupt */
143 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR,
144 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ);
145
146 return 0;
147 }
148
a6xx_hfi_wait_for_ack(struct a6xx_gmu * gmu,u32 id,u32 seqnum,u32 * payload,u32 payload_size)149 static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum,
150 u32 *payload, u32 payload_size)
151 {
152 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE];
153 int ret;
154
155 ret = a6xx_hfi_wait_for_msg_interrupt(gmu, id, seqnum);
156 if (ret)
157 return ret;
158
159 for (;;) {
160 struct a6xx_hfi_msg_response resp;
161
162 /* Get the next packet */
163 ret = a6xx_hfi_queue_read(gmu, queue, (u32 *) &resp,
164 sizeof(resp) >> 2);
165
166 /* If the queue is empty, there may have been previous missed
167 * responses that preceded the response to our packet. Wait
168 * further before we give up.
169 */
170 if (!ret) {
171 ret = a6xx_hfi_wait_for_msg_interrupt(gmu, id, seqnum);
172 if (ret) {
173 DRM_DEV_ERROR(gmu->dev,
174 "The HFI response queue is unexpectedly empty\n");
175 return ret;
176 }
177 continue;
178 }
179
180 if (HFI_HEADER_ID(resp.header) == HFI_F2H_MSG_ERROR) {
181 struct a6xx_hfi_msg_error *error =
182 (struct a6xx_hfi_msg_error *) &resp;
183
184 DRM_DEV_ERROR(gmu->dev, "GMU firmware error %d\n",
185 error->code);
186 continue;
187 }
188
189 if (seqnum != HFI_HEADER_SEQNUM(resp.ret_header)) {
190 DRM_DEV_ERROR(gmu->dev,
191 "Unexpected message id %d on the response queue\n",
192 HFI_HEADER_SEQNUM(resp.ret_header));
193 continue;
194 }
195
196 if (resp.error) {
197 DRM_DEV_ERROR(gmu->dev,
198 "Message %s id %d returned error %d\n",
199 a6xx_hfi_msg_id[id], seqnum, resp.error);
200 return -EINVAL;
201 }
202
203 /* All is well, copy over the buffer */
204 if (payload && payload_size)
205 memcpy(payload, resp.payload,
206 min_t(u32, payload_size, sizeof(resp.payload)));
207
208 return 0;
209 }
210 }
211
a6xx_hfi_send_msg(struct a6xx_gmu * gmu,int id,void * data,u32 size,u32 * payload,u32 payload_size)212 static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id,
213 void *data, u32 size, u32 *payload, u32 payload_size)
214 {
215 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE];
216 int ret, dwords = size >> 2;
217 u32 seqnum;
218
219 seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;
220
221 /* First dword of the message is the message header - fill it in */
222 *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) |
223 (dwords << 8) | id;
224
225 ret = a6xx_hfi_queue_write(gmu, queue, data, dwords);
226 if (ret) {
227 DRM_DEV_ERROR(gmu->dev, "Unable to send message %s id %d\n",
228 a6xx_hfi_msg_id[id], seqnum);
229 return ret;
230 }
231
232 return a6xx_hfi_wait_for_ack(gmu, id, seqnum, payload, payload_size);
233 }
234
a6xx_hfi_send_gmu_init(struct a6xx_gmu * gmu,int boot_state)235 static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
236 {
237 struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
238
239 msg.dbg_buffer_addr = (u32) gmu->debug.iova;
240 msg.dbg_buffer_size = (u32) gmu->debug.size;
241 msg.boot_state = boot_state;
242
243 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
244 NULL, 0);
245 }
246
a6xx_hfi_get_fw_version(struct a6xx_gmu * gmu,u32 * version)247 static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
248 {
249 struct a6xx_hfi_msg_fw_version msg = { 0 };
250
251 /* Currently supporting version 1.10 */
252 msg.supported_version = (1 << 28) | (1 << 19) | (1 << 17);
253
254 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg),
255 version, sizeof(*version));
256 }
257
a6xx_hfi_send_perf_table_v1(struct a6xx_gmu * gmu)258 static int a6xx_hfi_send_perf_table_v1(struct a6xx_gmu *gmu)
259 {
260 struct a6xx_hfi_msg_perf_table_v1 msg = { 0 };
261 int i;
262
263 msg.num_gpu_levels = gmu->nr_gpu_freqs;
264 msg.num_gmu_levels = gmu->nr_gmu_freqs;
265
266 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
267 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
268 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
269 }
270
271 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
272 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
273 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
274 }
275
276 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
277 NULL, 0);
278 }
279
a8xx_hfi_send_perf_table(struct a6xx_gmu * gmu)280 static int a8xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
281 {
282 unsigned int num_gx_votes = 3, num_cx_votes = 2;
283 struct a6xx_hfi_table_entry *entry;
284 struct a6xx_hfi_table *tbl;
285 int ret, i;
286 u32 size;
287
288 size = sizeof(*tbl) + (2 * sizeof(tbl->entry[0])) +
289 (gmu->nr_gpu_freqs * num_gx_votes * sizeof(gmu->gx_arc_votes[0])) +
290 (gmu->nr_gmu_freqs * num_cx_votes * sizeof(gmu->cx_arc_votes[0]));
291 tbl = kzalloc(size, GFP_KERNEL);
292 if (!tbl)
293 return -ENOMEM;
294 tbl->type = HFI_TABLE_GPU_PERF;
295
296 /* First fill GX votes */
297 entry = &tbl->entry[0];
298 entry->count = gmu->nr_gpu_freqs;
299 entry->stride = num_gx_votes;
300
301 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
302 unsigned int base = i * entry->stride;
303
304 entry->data[base+0] = gmu->gx_arc_votes[i];
305 entry->data[base+1] = gmu->dep_arc_votes[i];
306 entry->data[base+2] = gmu->gpu_freqs[i] / 1000;
307 }
308
309 /* Then fill CX votes */
310 entry = (struct a6xx_hfi_table_entry *)
311 &tbl->entry[0].data[gmu->nr_gpu_freqs * num_gx_votes];
312
313 entry->count = gmu->nr_gmu_freqs;
314 entry->stride = num_cx_votes;
315
316 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
317 unsigned int base = i * entry->stride;
318
319 entry->data[base] = gmu->cx_arc_votes[i];
320 entry->data[base+1] = gmu->gmu_freqs[i] / 1000;
321 }
322
323 ret = a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TABLE, tbl, size, NULL, 0);
324
325 kfree(tbl);
326 return ret;
327 }
328
a6xx_hfi_send_perf_table(struct a6xx_gmu * gmu)329 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
330 {
331 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
332 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
333 struct a6xx_hfi_msg_perf_table msg = { 0 };
334 int i;
335
336 if (adreno_is_a8xx(adreno_gpu))
337 return a8xx_hfi_send_perf_table(gmu);
338
339 msg.num_gpu_levels = gmu->nr_gpu_freqs;
340 msg.num_gmu_levels = gmu->nr_gmu_freqs;
341
342 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
343 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
344 msg.gx_votes[i].acd = 0xffffffff;
345 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
346 }
347
348 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
349 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
350 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
351 }
352
353 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
354 NULL, 0);
355 }
356
a6xx_generate_bw_table(const struct a6xx_info * info,struct a6xx_gmu * gmu,struct a6xx_hfi_msg_bw_table * msg)357 static void a6xx_generate_bw_table(const struct a6xx_info *info, struct a6xx_gmu *gmu,
358 struct a6xx_hfi_msg_bw_table *msg)
359 {
360 unsigned int i, j;
361
362 for (i = 0; i < GMU_MAX_BCMS; i++) {
363 if (!info->bcms[i].name)
364 break;
365 msg->ddr_cmds_addrs[i] = cmd_db_read_addr(info->bcms[i].name);
366 }
367 msg->ddr_cmds_num = i;
368
369 for (i = 0; i < gmu->nr_gpu_bws; ++i)
370 for (j = 0; j < msg->ddr_cmds_num; j++)
371 msg->ddr_cmds_data[i][j] = gmu->gpu_ib_votes[i][j];
372 msg->bw_level_num = gmu->nr_gpu_bws;
373
374 /* Compute the wait bitmask with each BCM having the commit bit */
375 msg->ddr_wait_bitmask = 0;
376 for (j = 0; j < msg->ddr_cmds_num; j++)
377 if (msg->ddr_cmds_data[0][j] & BCM_TCS_CMD_COMMIT_MASK)
378 msg->ddr_wait_bitmask |= BIT(j);
379
380 /*
381 * These are the CX (CNOC) votes - these are used by the GMU
382 * The 'CN0' BCM is used on all targets, and votes are basically
383 * 'off' and 'on' states with first bit to enable the path.
384 */
385
386 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");
387 msg->cnoc_cmds_num = 1;
388
389 msg->cnoc_cmds_data[0][0] = BCM_TCS_CMD(true, false, 0, 0);
390 msg->cnoc_cmds_data[1][0] = BCM_TCS_CMD(true, true, 0, BIT(0));
391
392 /* Compute the wait bitmask with each BCM having the commit bit */
393 msg->cnoc_wait_bitmask = 0;
394 for (j = 0; j < msg->cnoc_cmds_num; j++)
395 if (msg->cnoc_cmds_data[0][j] & BCM_TCS_CMD_COMMIT_MASK)
396 msg->cnoc_wait_bitmask |= BIT(j);
397 }
398
a618_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)399 static void a618_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
400 {
401 /* Send a single "off" entry since the 618 GMU doesn't do bus scaling */
402 msg->bw_level_num = 1;
403
404 msg->ddr_cmds_num = 3;
405 msg->ddr_wait_bitmask = 0x01;
406
407 msg->ddr_cmds_addrs[0] = 0x50000;
408 msg->ddr_cmds_addrs[1] = 0x5003c;
409 msg->ddr_cmds_addrs[2] = 0x5000c;
410
411 msg->ddr_cmds_data[0][0] = 0x40000000;
412 msg->ddr_cmds_data[0][1] = 0x40000000;
413 msg->ddr_cmds_data[0][2] = 0x40000000;
414
415 /*
416 * These are the CX (CNOC) votes - these are used by the GMU but the
417 * votes are known and fixed for the target
418 */
419 msg->cnoc_cmds_num = 1;
420 msg->cnoc_wait_bitmask = 0x01;
421
422 msg->cnoc_cmds_addrs[0] = 0x5007c;
423 msg->cnoc_cmds_data[0][0] = 0x40000000;
424 msg->cnoc_cmds_data[1][0] = 0x60000001;
425 }
426
a619_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)427 static void a619_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
428 {
429 msg->bw_level_num = 13;
430
431 msg->ddr_cmds_num = 3;
432 msg->ddr_wait_bitmask = 0x0;
433
434 msg->ddr_cmds_addrs[0] = 0x50000;
435 msg->ddr_cmds_addrs[1] = 0x50004;
436 msg->ddr_cmds_addrs[2] = 0x50080;
437
438 msg->ddr_cmds_data[0][0] = 0x40000000;
439 msg->ddr_cmds_data[0][1] = 0x40000000;
440 msg->ddr_cmds_data[0][2] = 0x40000000;
441 msg->ddr_cmds_data[1][0] = 0x6000030c;
442 msg->ddr_cmds_data[1][1] = 0x600000db;
443 msg->ddr_cmds_data[1][2] = 0x60000008;
444 msg->ddr_cmds_data[2][0] = 0x60000618;
445 msg->ddr_cmds_data[2][1] = 0x600001b6;
446 msg->ddr_cmds_data[2][2] = 0x60000008;
447 msg->ddr_cmds_data[3][0] = 0x60000925;
448 msg->ddr_cmds_data[3][1] = 0x60000291;
449 msg->ddr_cmds_data[3][2] = 0x60000008;
450 msg->ddr_cmds_data[4][0] = 0x60000dc1;
451 msg->ddr_cmds_data[4][1] = 0x600003dc;
452 msg->ddr_cmds_data[4][2] = 0x60000008;
453 msg->ddr_cmds_data[5][0] = 0x600010ad;
454 msg->ddr_cmds_data[5][1] = 0x600004ae;
455 msg->ddr_cmds_data[5][2] = 0x60000008;
456 msg->ddr_cmds_data[6][0] = 0x600014c3;
457 msg->ddr_cmds_data[6][1] = 0x600005d4;
458 msg->ddr_cmds_data[6][2] = 0x60000008;
459 msg->ddr_cmds_data[7][0] = 0x6000176a;
460 msg->ddr_cmds_data[7][1] = 0x60000693;
461 msg->ddr_cmds_data[7][2] = 0x60000008;
462 msg->ddr_cmds_data[8][0] = 0x60001f01;
463 msg->ddr_cmds_data[8][1] = 0x600008b5;
464 msg->ddr_cmds_data[8][2] = 0x60000008;
465 msg->ddr_cmds_data[9][0] = 0x60002940;
466 msg->ddr_cmds_data[9][1] = 0x60000b95;
467 msg->ddr_cmds_data[9][2] = 0x60000008;
468 msg->ddr_cmds_data[10][0] = 0x60002f68;
469 msg->ddr_cmds_data[10][1] = 0x60000d50;
470 msg->ddr_cmds_data[10][2] = 0x60000008;
471 msg->ddr_cmds_data[11][0] = 0x60003700;
472 msg->ddr_cmds_data[11][1] = 0x60000f71;
473 msg->ddr_cmds_data[11][2] = 0x60000008;
474 msg->ddr_cmds_data[12][0] = 0x60003fce;
475 msg->ddr_cmds_data[12][1] = 0x600011ea;
476 msg->ddr_cmds_data[12][2] = 0x60000008;
477
478 msg->cnoc_cmds_num = 1;
479 msg->cnoc_wait_bitmask = 0x0;
480
481 msg->cnoc_cmds_addrs[0] = 0x50054;
482
483 msg->cnoc_cmds_data[0][0] = 0x40000000;
484 }
485
a640_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)486 static void a640_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
487 {
488 /*
489 * Send a single "off" entry just to get things running
490 * TODO: bus scaling
491 */
492 msg->bw_level_num = 1;
493
494 msg->ddr_cmds_num = 3;
495 msg->ddr_wait_bitmask = 0x01;
496
497 msg->ddr_cmds_addrs[0] = 0x50000;
498 msg->ddr_cmds_addrs[1] = 0x5003c;
499 msg->ddr_cmds_addrs[2] = 0x5000c;
500
501 msg->ddr_cmds_data[0][0] = 0x40000000;
502 msg->ddr_cmds_data[0][1] = 0x40000000;
503 msg->ddr_cmds_data[0][2] = 0x40000000;
504
505 /*
506 * These are the CX (CNOC) votes - these are used by the GMU but the
507 * votes are known and fixed for the target
508 */
509 msg->cnoc_cmds_num = 3;
510 msg->cnoc_wait_bitmask = 0x01;
511
512 msg->cnoc_cmds_addrs[0] = 0x50034;
513 msg->cnoc_cmds_addrs[1] = 0x5007c;
514 msg->cnoc_cmds_addrs[2] = 0x5004c;
515
516 msg->cnoc_cmds_data[0][0] = 0x40000000;
517 msg->cnoc_cmds_data[0][1] = 0x00000000;
518 msg->cnoc_cmds_data[0][2] = 0x40000000;
519
520 msg->cnoc_cmds_data[1][0] = 0x60000001;
521 msg->cnoc_cmds_data[1][1] = 0x20000001;
522 msg->cnoc_cmds_data[1][2] = 0x60000001;
523 }
524
a650_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)525 static void a650_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
526 {
527 /*
528 * Send a single "off" entry just to get things running
529 * TODO: bus scaling
530 */
531 msg->bw_level_num = 1;
532
533 msg->ddr_cmds_num = 3;
534 msg->ddr_wait_bitmask = 0x01;
535
536 msg->ddr_cmds_addrs[0] = 0x50000;
537 msg->ddr_cmds_addrs[1] = 0x50004;
538 msg->ddr_cmds_addrs[2] = 0x5007c;
539
540 msg->ddr_cmds_data[0][0] = 0x40000000;
541 msg->ddr_cmds_data[0][1] = 0x40000000;
542 msg->ddr_cmds_data[0][2] = 0x40000000;
543
544 /*
545 * These are the CX (CNOC) votes - these are used by the GMU but the
546 * votes are known and fixed for the target
547 */
548 msg->cnoc_cmds_num = 1;
549 msg->cnoc_wait_bitmask = 0x01;
550
551 msg->cnoc_cmds_addrs[0] = 0x500a4;
552 msg->cnoc_cmds_data[0][0] = 0x40000000;
553 msg->cnoc_cmds_data[1][0] = 0x60000001;
554 }
555
a690_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)556 static void a690_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
557 {
558 /*
559 * Send a single "off" entry just to get things running
560 * TODO: bus scaling
561 */
562 msg->bw_level_num = 1;
563
564 msg->ddr_cmds_num = 3;
565 msg->ddr_wait_bitmask = 0x01;
566
567 msg->ddr_cmds_addrs[0] = 0x50004;
568 msg->ddr_cmds_addrs[1] = 0x50000;
569 msg->ddr_cmds_addrs[2] = 0x500ac;
570
571 msg->ddr_cmds_data[0][0] = 0x40000000;
572 msg->ddr_cmds_data[0][1] = 0x40000000;
573 msg->ddr_cmds_data[0][2] = 0x40000000;
574
575 /*
576 * These are the CX (CNOC) votes - these are used by the GMU but the
577 * votes are known and fixed for the target
578 */
579 msg->cnoc_cmds_num = 1;
580 msg->cnoc_wait_bitmask = 0x01;
581
582 msg->cnoc_cmds_addrs[0] = 0x5003c;
583 msg->cnoc_cmds_data[0][0] = 0x40000000;
584 msg->cnoc_cmds_data[1][0] = 0x60000001;
585 }
586
a660_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)587 static void a660_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
588 {
589 /*
590 * Send a single "off" entry just to get things running
591 * TODO: bus scaling
592 */
593 msg->bw_level_num = 1;
594
595 msg->ddr_cmds_num = 3;
596 msg->ddr_wait_bitmask = 0x01;
597
598 msg->ddr_cmds_addrs[0] = 0x50004;
599 msg->ddr_cmds_addrs[1] = 0x500a0;
600 msg->ddr_cmds_addrs[2] = 0x50000;
601
602 msg->ddr_cmds_data[0][0] = 0x40000000;
603 msg->ddr_cmds_data[0][1] = 0x40000000;
604 msg->ddr_cmds_data[0][2] = 0x40000000;
605
606 /*
607 * These are the CX (CNOC) votes - these are used by the GMU but the
608 * votes are known and fixed for the target
609 */
610 msg->cnoc_cmds_num = 1;
611 msg->cnoc_wait_bitmask = 0x01;
612
613 msg->cnoc_cmds_addrs[0] = 0x50070;
614 msg->cnoc_cmds_data[0][0] = 0x40000000;
615 msg->cnoc_cmds_data[1][0] = 0x60000001;
616 }
617
a663_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)618 static void a663_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
619 {
620 /*
621 * Send a single "off" entry just to get things running
622 * TODO: bus scaling
623 */
624 msg->bw_level_num = 1;
625
626 msg->ddr_cmds_num = 3;
627 msg->ddr_wait_bitmask = 0x07;
628
629 msg->ddr_cmds_addrs[0] = 0x50004;
630 msg->ddr_cmds_addrs[1] = 0x50000;
631 msg->ddr_cmds_addrs[2] = 0x500b4;
632
633 msg->ddr_cmds_data[0][0] = 0x40000000;
634 msg->ddr_cmds_data[0][1] = 0x40000000;
635 msg->ddr_cmds_data[0][2] = 0x40000000;
636
637 /*
638 * These are the CX (CNOC) votes - these are used by the GMU but the
639 * votes are known and fixed for the target
640 */
641 msg->cnoc_cmds_num = 1;
642 msg->cnoc_wait_bitmask = 0x01;
643
644 msg->cnoc_cmds_addrs[0] = 0x50058;
645 msg->cnoc_cmds_data[0][0] = 0x40000000;
646 msg->cnoc_cmds_data[1][0] = 0x60000001;
647 }
648
adreno_7c3_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)649 static void adreno_7c3_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
650 {
651 /*
652 * Send a single "off" entry just to get things running
653 * TODO: bus scaling
654 */
655 msg->bw_level_num = 1;
656
657 msg->ddr_cmds_num = 3;
658 msg->ddr_wait_bitmask = 0x07;
659
660 msg->ddr_cmds_addrs[0] = 0x50004;
661 msg->ddr_cmds_addrs[1] = 0x50000;
662 msg->ddr_cmds_addrs[2] = 0x50088;
663
664 msg->ddr_cmds_data[0][0] = 0x40000000;
665 msg->ddr_cmds_data[0][1] = 0x40000000;
666 msg->ddr_cmds_data[0][2] = 0x40000000;
667
668 /*
669 * These are the CX (CNOC) votes - these are used by the GMU but the
670 * votes are known and fixed for the target
671 */
672 msg->cnoc_cmds_num = 1;
673 msg->cnoc_wait_bitmask = 0x01;
674
675 msg->cnoc_cmds_addrs[0] = 0x5006c;
676 msg->cnoc_cmds_data[0][0] = 0x40000000;
677 msg->cnoc_cmds_data[1][0] = 0x60000001;
678 }
679
a730_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)680 static void a730_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
681 {
682 msg->bw_level_num = 12;
683
684 msg->ddr_cmds_num = 3;
685 msg->ddr_wait_bitmask = 0x7;
686
687 msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0");
688 msg->ddr_cmds_addrs[1] = cmd_db_read_addr("MC0");
689 msg->ddr_cmds_addrs[2] = cmd_db_read_addr("ACV");
690
691 msg->ddr_cmds_data[0][0] = 0x40000000;
692 msg->ddr_cmds_data[0][1] = 0x40000000;
693 msg->ddr_cmds_data[0][2] = 0x40000000;
694 msg->ddr_cmds_data[1][0] = 0x600002e8;
695 msg->ddr_cmds_data[1][1] = 0x600003d0;
696 msg->ddr_cmds_data[1][2] = 0x60000008;
697 msg->ddr_cmds_data[2][0] = 0x6000068d;
698 msg->ddr_cmds_data[2][1] = 0x6000089a;
699 msg->ddr_cmds_data[2][2] = 0x60000008;
700 msg->ddr_cmds_data[3][0] = 0x600007f2;
701 msg->ddr_cmds_data[3][1] = 0x60000a6e;
702 msg->ddr_cmds_data[3][2] = 0x60000008;
703 msg->ddr_cmds_data[4][0] = 0x600009e5;
704 msg->ddr_cmds_data[4][1] = 0x60000cfd;
705 msg->ddr_cmds_data[4][2] = 0x60000008;
706 msg->ddr_cmds_data[5][0] = 0x60000b29;
707 msg->ddr_cmds_data[5][1] = 0x60000ea6;
708 msg->ddr_cmds_data[5][2] = 0x60000008;
709 msg->ddr_cmds_data[6][0] = 0x60001698;
710 msg->ddr_cmds_data[6][1] = 0x60001da8;
711 msg->ddr_cmds_data[6][2] = 0x60000008;
712 msg->ddr_cmds_data[7][0] = 0x600018d2;
713 msg->ddr_cmds_data[7][1] = 0x60002093;
714 msg->ddr_cmds_data[7][2] = 0x60000008;
715 msg->ddr_cmds_data[8][0] = 0x60001e66;
716 msg->ddr_cmds_data[8][1] = 0x600027e6;
717 msg->ddr_cmds_data[8][2] = 0x60000008;
718 msg->ddr_cmds_data[9][0] = 0x600027c2;
719 msg->ddr_cmds_data[9][1] = 0x6000342f;
720 msg->ddr_cmds_data[9][2] = 0x60000008;
721 msg->ddr_cmds_data[10][0] = 0x60002e71;
722 msg->ddr_cmds_data[10][1] = 0x60003cf5;
723 msg->ddr_cmds_data[10][2] = 0x60000008;
724 msg->ddr_cmds_data[11][0] = 0x600030ae;
725 msg->ddr_cmds_data[11][1] = 0x60003fe5;
726 msg->ddr_cmds_data[11][2] = 0x60000008;
727
728 msg->cnoc_cmds_num = 1;
729 msg->cnoc_wait_bitmask = 0x1;
730
731 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");
732 msg->cnoc_cmds_data[0][0] = 0x40000000;
733 msg->cnoc_cmds_data[1][0] = 0x60000001;
734 }
735
a740_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)736 static void a740_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
737 {
738 msg->bw_level_num = 1;
739
740 msg->ddr_cmds_num = 3;
741 msg->ddr_wait_bitmask = 0x7;
742
743 msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0");
744 msg->ddr_cmds_addrs[1] = cmd_db_read_addr("MC0");
745 msg->ddr_cmds_addrs[2] = cmd_db_read_addr("ACV");
746
747 msg->ddr_cmds_data[0][0] = 0x40000000;
748 msg->ddr_cmds_data[0][1] = 0x40000000;
749 msg->ddr_cmds_data[0][2] = 0x40000000;
750
751 /* TODO: add a proper dvfs table */
752
753 msg->cnoc_cmds_num = 1;
754 msg->cnoc_wait_bitmask = 0x1;
755
756 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");
757 msg->cnoc_cmds_data[0][0] = 0x40000000;
758 msg->cnoc_cmds_data[1][0] = 0x60000001;
759 }
760
a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)761 static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
762 {
763 /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */
764 msg->bw_level_num = 1;
765
766 msg->ddr_cmds_num = 3;
767 msg->ddr_wait_bitmask = 0x07;
768
769 msg->ddr_cmds_addrs[0] = 0x50000;
770 msg->ddr_cmds_addrs[1] = 0x5005c;
771 msg->ddr_cmds_addrs[2] = 0x5000c;
772
773 msg->ddr_cmds_data[0][0] = 0x40000000;
774 msg->ddr_cmds_data[0][1] = 0x40000000;
775 msg->ddr_cmds_data[0][2] = 0x40000000;
776
777 /*
778 * These are the CX (CNOC) votes. This is used but the values for the
779 * sdm845 GMU are known and fixed so we can hard code them.
780 */
781
782 msg->cnoc_cmds_num = 3;
783 msg->cnoc_wait_bitmask = 0x05;
784
785 msg->cnoc_cmds_addrs[0] = 0x50034;
786 msg->cnoc_cmds_addrs[1] = 0x5007c;
787 msg->cnoc_cmds_addrs[2] = 0x5004c;
788
789 msg->cnoc_cmds_data[0][0] = 0x40000000;
790 msg->cnoc_cmds_data[0][1] = 0x00000000;
791 msg->cnoc_cmds_data[0][2] = 0x40000000;
792
793 msg->cnoc_cmds_data[1][0] = 0x60000001;
794 msg->cnoc_cmds_data[1][1] = 0x20000001;
795 msg->cnoc_cmds_data[1][2] = 0x60000001;
796 }
797
798
a6xx_hfi_send_bw_table(struct a6xx_gmu * gmu)799 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
800 {
801 struct a6xx_hfi_msg_bw_table *msg;
802 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
803 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
804 const struct a6xx_info *info = adreno_gpu->info->a6xx;
805
806 if (gmu->bw_table)
807 goto send;
808
809 msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);
810 if (!msg)
811 return -ENOMEM;
812
813 if (info->bcms && gmu->nr_gpu_bws > 1)
814 a6xx_generate_bw_table(info, gmu, msg);
815 else if (adreno_is_a618(adreno_gpu))
816 a618_build_bw_table(msg);
817 else if (adreno_is_a619(adreno_gpu))
818 a619_build_bw_table(msg);
819 else if (adreno_is_a640_family(adreno_gpu))
820 a640_build_bw_table(msg);
821 else if (adreno_is_a650(adreno_gpu))
822 a650_build_bw_table(msg);
823 else if (adreno_is_7c3(adreno_gpu))
824 adreno_7c3_build_bw_table(msg);
825 else if (adreno_is_a660(adreno_gpu))
826 a660_build_bw_table(msg);
827 else if (adreno_is_a663(adreno_gpu))
828 a663_build_bw_table(msg);
829 else if (adreno_is_a690(adreno_gpu))
830 a690_build_bw_table(msg);
831 else if (adreno_is_a730(adreno_gpu))
832 a730_build_bw_table(msg);
833 else if (adreno_is_a740_family(adreno_gpu))
834 a740_build_bw_table(msg);
835 else
836 a6xx_build_bw_table(msg);
837
838 gmu->bw_table = msg;
839
840 send:
841 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)),
842 NULL, 0);
843 }
844
a6xx_hfi_feature_ctrl_msg(struct a6xx_gmu * gmu,u32 feature,u32 enable,u32 data)845 static int a6xx_hfi_feature_ctrl_msg(struct a6xx_gmu *gmu, u32 feature, u32 enable, u32 data)
846 {
847 struct a6xx_hfi_msg_feature_ctrl msg = {
848 .feature = feature,
849 .enable = enable,
850 .data = data,
851 };
852
853 return a6xx_hfi_send_msg(gmu, HFI_H2F_FEATURE_CTRL, &msg, sizeof(msg), NULL, 0);
854 }
855
856 #define IFPC_LONG_HYST 0x1680
857
a6xx_hfi_enable_ifpc(struct a6xx_gmu * gmu)858 static int a6xx_hfi_enable_ifpc(struct a6xx_gmu *gmu)
859 {
860 if (gmu->idle_level != GMU_IDLE_STATE_IFPC)
861 return 0;
862
863 return a6xx_hfi_feature_ctrl_msg(gmu, HFI_FEATURE_IFPC, 1, IFPC_LONG_HYST);
864 }
865
a6xx_hfi_enable_acd(struct a6xx_gmu * gmu)866 static int a6xx_hfi_enable_acd(struct a6xx_gmu *gmu)
867 {
868 struct a6xx_hfi_acd_table *acd_table = &gmu->acd_table;
869 int ret;
870
871 if (!acd_table->enable_by_level)
872 return 0;
873
874 /* Enable ACD feature at GMU */
875 ret = a6xx_hfi_feature_ctrl_msg(gmu, HFI_FEATURE_ACD, 1, 0);
876 if (ret) {
877 DRM_DEV_ERROR(gmu->dev, "Unable to enable ACD (%d)\n", ret);
878 return ret;
879 }
880
881 /* Send ACD table to GMU */
882 ret = a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_ACD, acd_table, sizeof(*acd_table), NULL, 0);
883 if (ret) {
884 DRM_DEV_ERROR(gmu->dev, "Unable to ACD table (%d)\n", ret);
885 return ret;
886 }
887
888 return 0;
889 }
890
a6xx_hfi_send_test(struct a6xx_gmu * gmu)891 static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
892 {
893 struct a6xx_hfi_msg_test msg = { 0 };
894
895 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg),
896 NULL, 0);
897 }
898
a6xx_hfi_send_start(struct a6xx_gmu * gmu)899 static int a6xx_hfi_send_start(struct a6xx_gmu *gmu)
900 {
901 struct a6xx_hfi_msg_start msg = { 0 };
902
903 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_START, &msg, sizeof(msg),
904 NULL, 0);
905 }
906
a6xx_hfi_send_core_fw_start(struct a6xx_gmu * gmu)907 static int a6xx_hfi_send_core_fw_start(struct a6xx_gmu *gmu)
908 {
909 struct a6xx_hfi_msg_core_fw_start msg = { 0 };
910
911 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_CORE_FW_START, &msg,
912 sizeof(msg), NULL, 0);
913 }
914
a6xx_hfi_set_freq(struct a6xx_gmu * gmu,u32 freq_index,u32 bw_index)915 int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, u32 freq_index, u32 bw_index)
916 {
917 struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 };
918
919 msg.ack_type = 1; /* blocking */
920 msg.freq = freq_index;
921 msg.bw = bw_index;
922
923 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_GX_BW_PERF_VOTE, &msg,
924 sizeof(msg), NULL, 0);
925 }
926
a6xx_hfi_send_prep_slumber(struct a6xx_gmu * gmu)927 int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu)
928 {
929 struct a6xx_hfi_prep_slumber_cmd msg = { 0 };
930
931 /* TODO: should freq and bw fields be non-zero ? */
932
933 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PREPARE_SLUMBER, &msg,
934 sizeof(msg), NULL, 0);
935 }
936
a6xx_hfi_start_v1(struct a6xx_gmu * gmu,int boot_state)937 static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu, int boot_state)
938 {
939 int ret;
940
941 ret = a6xx_hfi_send_gmu_init(gmu, boot_state);
942 if (ret)
943 return ret;
944
945 ret = a6xx_hfi_get_fw_version(gmu, NULL);
946 if (ret)
947 return ret;
948
949 /*
950 * We have to get exchange version numbers per the sequence but at this
951 * point th kernel driver doesn't need to know the exact version of
952 * the GMU firmware
953 */
954
955 ret = a6xx_hfi_send_perf_table_v1(gmu);
956 if (ret)
957 return ret;
958
959 ret = a6xx_hfi_send_bw_table(gmu);
960 if (ret)
961 return ret;
962
963 /*
964 * Let the GMU know that there won't be any more HFI messages until next
965 * boot
966 */
967 a6xx_hfi_send_test(gmu);
968
969 return 0;
970 }
971
a6xx_hfi_start(struct a6xx_gmu * gmu,int boot_state)972 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
973 {
974 int ret;
975
976 if (gmu->legacy)
977 return a6xx_hfi_start_v1(gmu, boot_state);
978
979
980 ret = a6xx_hfi_send_perf_table(gmu);
981 if (ret)
982 return ret;
983
984 ret = a6xx_hfi_send_bw_table(gmu);
985 if (ret)
986 return ret;
987
988 ret = a6xx_hfi_enable_acd(gmu);
989 if (ret)
990 return ret;
991
992 ret = a6xx_hfi_enable_ifpc(gmu);
993 if (ret)
994 return ret;
995
996 ret = a6xx_hfi_send_core_fw_start(gmu);
997 if (ret)
998 return ret;
999
1000 /*
1001 * Downstream driver sends this in its "a6xx_hw_init" equivalent,
1002 * but seems to be no harm in sending it here
1003 */
1004 ret = a6xx_hfi_send_start(gmu);
1005 if (ret)
1006 return ret;
1007
1008 return 0;
1009 }
1010
a6xx_hfi_stop(struct a6xx_gmu * gmu)1011 void a6xx_hfi_stop(struct a6xx_gmu *gmu)
1012 {
1013 int i;
1014
1015 for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
1016 struct a6xx_hfi_queue *queue = &gmu->queues[i];
1017
1018 if (!queue->header)
1019 continue;
1020
1021 if (queue->header->read_index != queue->header->write_index)
1022 DRM_DEV_ERROR(gmu->dev, "HFI queue %d is not empty\n", i);
1023
1024 queue->header->read_index = 0;
1025 queue->header->write_index = 0;
1026
1027 memset(&queue->history, 0xff, sizeof(queue->history));
1028 queue->history_idx = 0;
1029 }
1030 }
1031
a6xx_hfi_queue_init(struct a6xx_hfi_queue * queue,struct a6xx_hfi_queue_header * header,void * virt,u64 iova,u32 id)1032 static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
1033 struct a6xx_hfi_queue_header *header, void *virt, u64 iova,
1034 u32 id)
1035 {
1036 spin_lock_init(&queue->lock);
1037 queue->header = header;
1038 queue->data = virt;
1039 atomic_set(&queue->seqnum, 0);
1040
1041 memset(&queue->history, 0xff, sizeof(queue->history));
1042 queue->history_idx = 0;
1043
1044 /* Set up the shared memory header */
1045 header->iova = iova;
1046 header->type = 10 << 8 | id;
1047 header->status = 1;
1048 header->size = SZ_4K >> 2;
1049 header->msg_size = 0;
1050 header->dropped = 0;
1051 header->rx_watermark = 1;
1052 header->tx_watermark = 1;
1053 header->rx_request = 1;
1054 header->tx_request = 0;
1055 header->read_index = 0;
1056 header->write_index = 0;
1057 }
1058
a6xx_hfi_init(struct a6xx_gmu * gmu)1059 void a6xx_hfi_init(struct a6xx_gmu *gmu)
1060 {
1061 struct a6xx_gmu_bo *hfi = &gmu->hfi;
1062 struct a6xx_hfi_queue_table_header *table = hfi->virt;
1063 struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
1064 int table_size, idx;
1065 u64 offset;
1066
1067 /*
1068 * The table size is the size of the table header plus all of the queue
1069 * headers
1070 */
1071 table_size = sizeof(*table);
1072 table_size += (ARRAY_SIZE(gmu->queues) *
1073 sizeof(struct a6xx_hfi_queue_header));
1074
1075 table->version = 0;
1076 table->size = table_size;
1077 /* First queue header is located immediately after the table header */
1078 table->qhdr0_offset = sizeof(*table) >> 2;
1079 table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2;
1080 table->num_queues = ARRAY_SIZE(gmu->queues);
1081 table->active_queues = ARRAY_SIZE(gmu->queues);
1082
1083 /* Command queue */
1084 idx = 0;
1085 offset = SZ_4K;
1086 a6xx_hfi_queue_init(&gmu->queues[idx], &headers[idx], hfi->virt + offset,
1087 hfi->iova + offset, 0);
1088
1089 /* GMU response queue */
1090 idx++;
1091 offset += SZ_4K;
1092 a6xx_hfi_queue_init(&gmu->queues[idx], &headers[idx], hfi->virt + offset,
1093 hfi->iova + offset, gmu->legacy ? 4 : 1);
1094
1095 /* GMU Debug queue */
1096 idx++;
1097 offset += SZ_4K;
1098 a6xx_hfi_queue_init(&gmu->queues[idx], &headers[idx], hfi->virt + offset,
1099 hfi->iova + offset, gmu->legacy ? 5 : 2);
1100
1101 WARN_ON(idx >= HFI_MAX_QUEUES);
1102 }
1103