1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */
3
4 #include <linux/completion.h>
5 #include <linux/circ_buf.h>
6 #include <linux/list.h>
7
8 #include <soc/qcom/cmd-db.h>
9 #include <soc/qcom/tcs.h>
10
11 #include "a6xx_gmu.h"
12 #include "a6xx_gmu.xml.h"
13 #include "a6xx_gpu.h"
14
15 #define HFI_MSG_ID(val) [val] = #val
16
17 static const char * const a6xx_hfi_msg_id[] = {
18 HFI_MSG_ID(HFI_H2F_MSG_INIT),
19 HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION),
20 HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE),
21 HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE),
22 HFI_MSG_ID(HFI_H2F_MSG_TEST),
23 HFI_MSG_ID(HFI_H2F_MSG_START),
24 HFI_MSG_ID(HFI_H2F_FEATURE_CTRL),
25 HFI_MSG_ID(HFI_H2F_MSG_CORE_FW_START),
26 HFI_MSG_ID(HFI_H2F_MSG_TABLE),
27 HFI_MSG_ID(HFI_H2F_MSG_GX_BW_PERF_VOTE),
28 HFI_MSG_ID(HFI_H2F_MSG_PREPARE_SLUMBER),
29 };
30
a6xx_hfi_queue_read(struct a6xx_gmu * gmu,struct a6xx_hfi_queue * queue,u32 * data,u32 dwords)31 static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu,
32 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
33 {
34 struct a6xx_hfi_queue_header *header = queue->header;
35 u32 i, hdr, index = header->read_index;
36
37 if (header->read_index == header->write_index) {
38 header->rx_request = 1;
39 return 0;
40 }
41
42 hdr = queue->data[index];
43
44 queue->history[(queue->history_idx++) % HFI_HISTORY_SZ] = index;
45
46 /*
47 * If we are to assume that the GMU firmware is in fact a rational actor
48 * and is programmed to not send us a larger response than we expect
49 * then we can also assume that if the header size is unexpectedly large
50 * that it is due to memory corruption and/or hardware failure. In this
51 * case the only reasonable course of action is to BUG() to help harden
52 * the failure.
53 */
54
55 BUG_ON(HFI_HEADER_SIZE(hdr) > dwords);
56
57 for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) {
58 data[i] = queue->data[index];
59 index = (index + 1) % header->size;
60 }
61
62 if (!gmu->legacy)
63 index = ALIGN(index, 4) % header->size;
64
65 header->read_index = index;
66 return HFI_HEADER_SIZE(hdr);
67 }
68
a6xx_hfi_queue_write(struct a6xx_gmu * gmu,struct a6xx_hfi_queue * queue,u32 * data,u32 dwords)69 static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu,
70 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords)
71 {
72 struct a6xx_hfi_queue_header *header = queue->header;
73 u32 i, space, index = header->write_index;
74
75 spin_lock(&queue->lock);
76
77 space = CIRC_SPACE(header->write_index, header->read_index,
78 header->size);
79 if (space < dwords) {
80 header->dropped++;
81 spin_unlock(&queue->lock);
82 return -ENOSPC;
83 }
84
85 queue->history[(queue->history_idx++) % HFI_HISTORY_SZ] = index;
86
87 for (i = 0; i < dwords; i++) {
88 queue->data[index] = data[i];
89 index = (index + 1) % header->size;
90 }
91
92 /* Cookify any non used data at the end of the write buffer */
93 if (!gmu->legacy) {
94 for (; index % 4; index = (index + 1) % header->size)
95 queue->data[index] = 0xfafafafa;
96 }
97
98 header->write_index = index;
99 spin_unlock(&queue->lock);
100
101 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01);
102 return 0;
103 }
104
a6xx_hfi_wait_for_msg_interrupt(struct a6xx_gmu * gmu,u32 id,u32 seqnum)105 static int a6xx_hfi_wait_for_msg_interrupt(struct a6xx_gmu *gmu, u32 id, u32 seqnum)
106 {
107 int ret;
108 u32 val;
109 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
110
111 do {
112 /* Wait for a response */
113 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val,
114 val & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 100, 1000000);
115
116 if (!ret)
117 break;
118
119 if (completion_done(&a6xx_gpu->base.fault_coredump_done))
120 break;
121
122 /* We may timeout because the GMU is temporarily wedged from
123 * pending faults from the GPU and we are taking a devcoredump.
124 * Wait until the MMU is resumed and try again.
125 */
126 wait_for_completion(&a6xx_gpu->base.fault_coredump_done);
127 } while (true);
128
129 if (ret) {
130 DRM_DEV_ERROR(gmu->dev,
131 "Message %s id %d timed out waiting for response\n",
132 a6xx_hfi_msg_id[id], seqnum);
133 return -ETIMEDOUT;
134 }
135
136 /* Clear the interrupt */
137 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR,
138 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ);
139
140 return 0;
141 }
142
a6xx_hfi_wait_for_ack(struct a6xx_gmu * gmu,u32 id,u32 seqnum,u32 * payload,u32 payload_size)143 static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum,
144 u32 *payload, u32 payload_size)
145 {
146 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE];
147 int ret;
148
149 ret = a6xx_hfi_wait_for_msg_interrupt(gmu, id, seqnum);
150 if (ret)
151 return ret;
152
153 for (;;) {
154 struct a6xx_hfi_msg_response resp;
155
156 /* Get the next packet */
157 ret = a6xx_hfi_queue_read(gmu, queue, (u32 *) &resp,
158 sizeof(resp) >> 2);
159
160 /* If the queue is empty, there may have been previous missed
161 * responses that preceded the response to our packet. Wait
162 * further before we give up.
163 */
164 if (!ret) {
165 ret = a6xx_hfi_wait_for_msg_interrupt(gmu, id, seqnum);
166 if (ret) {
167 DRM_DEV_ERROR(gmu->dev,
168 "The HFI response queue is unexpectedly empty\n");
169 return ret;
170 }
171 continue;
172 }
173
174 if (HFI_HEADER_ID(resp.header) == HFI_F2H_MSG_ERROR) {
175 struct a6xx_hfi_msg_error *error =
176 (struct a6xx_hfi_msg_error *) &resp;
177
178 DRM_DEV_ERROR(gmu->dev, "GMU firmware error %d\n",
179 error->code);
180 continue;
181 }
182
183 if (seqnum != HFI_HEADER_SEQNUM(resp.ret_header)) {
184 DRM_DEV_ERROR(gmu->dev,
185 "Unexpected message id %d on the response queue\n",
186 HFI_HEADER_SEQNUM(resp.ret_header));
187 continue;
188 }
189
190 if (resp.error) {
191 DRM_DEV_ERROR(gmu->dev,
192 "Message %s id %d returned error %d\n",
193 a6xx_hfi_msg_id[id], seqnum, resp.error);
194 return -EINVAL;
195 }
196
197 /* All is well, copy over the buffer */
198 if (payload && payload_size)
199 memcpy(payload, resp.payload,
200 min_t(u32, payload_size, sizeof(resp.payload)));
201
202 return 0;
203 }
204 }
205
a6xx_hfi_send_msg(struct a6xx_gmu * gmu,int id,void * data,u32 size,u32 * payload,u32 payload_size)206 static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id,
207 void *data, u32 size, u32 *payload, u32 payload_size)
208 {
209 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE];
210 int ret, dwords = size >> 2;
211 u32 seqnum;
212
213 seqnum = atomic_inc_return(&queue->seqnum) % 0xfff;
214
215 /* First dword of the message is the message header - fill it in */
216 *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) |
217 (dwords << 8) | id;
218
219 ret = a6xx_hfi_queue_write(gmu, queue, data, dwords);
220 if (ret) {
221 DRM_DEV_ERROR(gmu->dev, "Unable to send message %s id %d\n",
222 a6xx_hfi_msg_id[id], seqnum);
223 return ret;
224 }
225
226 return a6xx_hfi_wait_for_ack(gmu, id, seqnum, payload, payload_size);
227 }
228
a6xx_hfi_send_gmu_init(struct a6xx_gmu * gmu,int boot_state)229 static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state)
230 {
231 struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 };
232
233 msg.dbg_buffer_addr = (u32) gmu->debug.iova;
234 msg.dbg_buffer_size = (u32) gmu->debug.size;
235 msg.boot_state = boot_state;
236
237 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg),
238 NULL, 0);
239 }
240
a6xx_hfi_get_fw_version(struct a6xx_gmu * gmu,u32 * version)241 static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version)
242 {
243 struct a6xx_hfi_msg_fw_version msg = { 0 };
244
245 /* Currently supporting version 1.10 */
246 msg.supported_version = (1 << 28) | (1 << 19) | (1 << 17);
247
248 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg),
249 version, sizeof(*version));
250 }
251
a6xx_hfi_send_perf_table_v1(struct a6xx_gmu * gmu)252 static int a6xx_hfi_send_perf_table_v1(struct a6xx_gmu *gmu)
253 {
254 struct a6xx_hfi_msg_perf_table_v1 msg = { 0 };
255 int i;
256
257 msg.num_gpu_levels = gmu->nr_gpu_freqs;
258 msg.num_gmu_levels = gmu->nr_gmu_freqs;
259
260 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
261 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
262 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
263 }
264
265 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
266 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
267 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
268 }
269
270 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
271 NULL, 0);
272 }
273
a8xx_hfi_send_perf_table(struct a6xx_gmu * gmu)274 static int a8xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
275 {
276 unsigned int num_gx_votes = 3, num_cx_votes = 2;
277 struct a6xx_hfi_table_entry *entry;
278 struct a6xx_hfi_table *tbl;
279 int ret, i;
280 u32 size;
281
282 size = sizeof(*tbl) + (2 * sizeof(tbl->entry[0])) +
283 (gmu->nr_gpu_freqs * num_gx_votes * sizeof(gmu->gx_arc_votes[0])) +
284 (gmu->nr_gmu_freqs * num_cx_votes * sizeof(gmu->cx_arc_votes[0]));
285 tbl = kzalloc(size, GFP_KERNEL);
286 tbl->type = HFI_TABLE_GPU_PERF;
287
288 /* First fill GX votes */
289 entry = &tbl->entry[0];
290 entry->count = gmu->nr_gpu_freqs;
291 entry->stride = num_gx_votes;
292
293 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
294 unsigned int base = i * entry->stride;
295
296 entry->data[base+0] = gmu->gx_arc_votes[i];
297 entry->data[base+1] = gmu->dep_arc_votes[i];
298 entry->data[base+2] = gmu->gpu_freqs[i] / 1000;
299 }
300
301 /* Then fill CX votes */
302 entry = (struct a6xx_hfi_table_entry *)
303 &tbl->entry[0].data[gmu->nr_gpu_freqs * num_gx_votes];
304
305 entry->count = gmu->nr_gmu_freqs;
306 entry->stride = num_cx_votes;
307
308 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
309 unsigned int base = i * entry->stride;
310
311 entry->data[base] = gmu->cx_arc_votes[i];
312 entry->data[base+1] = gmu->gmu_freqs[i] / 1000;
313 }
314
315 ret = a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TABLE, tbl, size, NULL, 0);
316
317 kfree(tbl);
318 return ret;
319 }
320
a6xx_hfi_send_perf_table(struct a6xx_gmu * gmu)321 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu)
322 {
323 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
324 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
325 struct a6xx_hfi_msg_perf_table msg = { 0 };
326 int i;
327
328 if (adreno_is_a8xx(adreno_gpu))
329 return a8xx_hfi_send_perf_table(gmu);
330
331 msg.num_gpu_levels = gmu->nr_gpu_freqs;
332 msg.num_gmu_levels = gmu->nr_gmu_freqs;
333
334 for (i = 0; i < gmu->nr_gpu_freqs; i++) {
335 msg.gx_votes[i].vote = gmu->gx_arc_votes[i];
336 msg.gx_votes[i].acd = 0xffffffff;
337 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000;
338 }
339
340 for (i = 0; i < gmu->nr_gmu_freqs; i++) {
341 msg.cx_votes[i].vote = gmu->cx_arc_votes[i];
342 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000;
343 }
344
345 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg),
346 NULL, 0);
347 }
348
a6xx_generate_bw_table(const struct a6xx_info * info,struct a6xx_gmu * gmu,struct a6xx_hfi_msg_bw_table * msg)349 static void a6xx_generate_bw_table(const struct a6xx_info *info, struct a6xx_gmu *gmu,
350 struct a6xx_hfi_msg_bw_table *msg)
351 {
352 unsigned int i, j;
353
354 for (i = 0; i < GMU_MAX_BCMS; i++) {
355 if (!info->bcms[i].name)
356 break;
357 msg->ddr_cmds_addrs[i] = cmd_db_read_addr(info->bcms[i].name);
358 }
359 msg->ddr_cmds_num = i;
360
361 for (i = 0; i < gmu->nr_gpu_bws; ++i)
362 for (j = 0; j < msg->ddr_cmds_num; j++)
363 msg->ddr_cmds_data[i][j] = gmu->gpu_ib_votes[i][j];
364 msg->bw_level_num = gmu->nr_gpu_bws;
365
366 /* Compute the wait bitmask with each BCM having the commit bit */
367 msg->ddr_wait_bitmask = 0;
368 for (j = 0; j < msg->ddr_cmds_num; j++)
369 if (msg->ddr_cmds_data[0][j] & BCM_TCS_CMD_COMMIT_MASK)
370 msg->ddr_wait_bitmask |= BIT(j);
371
372 /*
373 * These are the CX (CNOC) votes - these are used by the GMU
374 * The 'CN0' BCM is used on all targets, and votes are basically
375 * 'off' and 'on' states with first bit to enable the path.
376 */
377
378 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");
379 msg->cnoc_cmds_num = 1;
380
381 msg->cnoc_cmds_data[0][0] = BCM_TCS_CMD(true, false, 0, 0);
382 msg->cnoc_cmds_data[1][0] = BCM_TCS_CMD(true, true, 0, BIT(0));
383
384 /* Compute the wait bitmask with each BCM having the commit bit */
385 msg->cnoc_wait_bitmask = 0;
386 for (j = 0; j < msg->cnoc_cmds_num; j++)
387 if (msg->cnoc_cmds_data[0][j] & BCM_TCS_CMD_COMMIT_MASK)
388 msg->cnoc_wait_bitmask |= BIT(j);
389 }
390
a618_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)391 static void a618_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
392 {
393 /* Send a single "off" entry since the 618 GMU doesn't do bus scaling */
394 msg->bw_level_num = 1;
395
396 msg->ddr_cmds_num = 3;
397 msg->ddr_wait_bitmask = 0x01;
398
399 msg->ddr_cmds_addrs[0] = 0x50000;
400 msg->ddr_cmds_addrs[1] = 0x5003c;
401 msg->ddr_cmds_addrs[2] = 0x5000c;
402
403 msg->ddr_cmds_data[0][0] = 0x40000000;
404 msg->ddr_cmds_data[0][1] = 0x40000000;
405 msg->ddr_cmds_data[0][2] = 0x40000000;
406
407 /*
408 * These are the CX (CNOC) votes - these are used by the GMU but the
409 * votes are known and fixed for the target
410 */
411 msg->cnoc_cmds_num = 1;
412 msg->cnoc_wait_bitmask = 0x01;
413
414 msg->cnoc_cmds_addrs[0] = 0x5007c;
415 msg->cnoc_cmds_data[0][0] = 0x40000000;
416 msg->cnoc_cmds_data[1][0] = 0x60000001;
417 }
418
a619_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)419 static void a619_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
420 {
421 msg->bw_level_num = 13;
422
423 msg->ddr_cmds_num = 3;
424 msg->ddr_wait_bitmask = 0x0;
425
426 msg->ddr_cmds_addrs[0] = 0x50000;
427 msg->ddr_cmds_addrs[1] = 0x50004;
428 msg->ddr_cmds_addrs[2] = 0x50080;
429
430 msg->ddr_cmds_data[0][0] = 0x40000000;
431 msg->ddr_cmds_data[0][1] = 0x40000000;
432 msg->ddr_cmds_data[0][2] = 0x40000000;
433 msg->ddr_cmds_data[1][0] = 0x6000030c;
434 msg->ddr_cmds_data[1][1] = 0x600000db;
435 msg->ddr_cmds_data[1][2] = 0x60000008;
436 msg->ddr_cmds_data[2][0] = 0x60000618;
437 msg->ddr_cmds_data[2][1] = 0x600001b6;
438 msg->ddr_cmds_data[2][2] = 0x60000008;
439 msg->ddr_cmds_data[3][0] = 0x60000925;
440 msg->ddr_cmds_data[3][1] = 0x60000291;
441 msg->ddr_cmds_data[3][2] = 0x60000008;
442 msg->ddr_cmds_data[4][0] = 0x60000dc1;
443 msg->ddr_cmds_data[4][1] = 0x600003dc;
444 msg->ddr_cmds_data[4][2] = 0x60000008;
445 msg->ddr_cmds_data[5][0] = 0x600010ad;
446 msg->ddr_cmds_data[5][1] = 0x600004ae;
447 msg->ddr_cmds_data[5][2] = 0x60000008;
448 msg->ddr_cmds_data[6][0] = 0x600014c3;
449 msg->ddr_cmds_data[6][1] = 0x600005d4;
450 msg->ddr_cmds_data[6][2] = 0x60000008;
451 msg->ddr_cmds_data[7][0] = 0x6000176a;
452 msg->ddr_cmds_data[7][1] = 0x60000693;
453 msg->ddr_cmds_data[7][2] = 0x60000008;
454 msg->ddr_cmds_data[8][0] = 0x60001f01;
455 msg->ddr_cmds_data[8][1] = 0x600008b5;
456 msg->ddr_cmds_data[8][2] = 0x60000008;
457 msg->ddr_cmds_data[9][0] = 0x60002940;
458 msg->ddr_cmds_data[9][1] = 0x60000b95;
459 msg->ddr_cmds_data[9][2] = 0x60000008;
460 msg->ddr_cmds_data[10][0] = 0x60002f68;
461 msg->ddr_cmds_data[10][1] = 0x60000d50;
462 msg->ddr_cmds_data[10][2] = 0x60000008;
463 msg->ddr_cmds_data[11][0] = 0x60003700;
464 msg->ddr_cmds_data[11][1] = 0x60000f71;
465 msg->ddr_cmds_data[11][2] = 0x60000008;
466 msg->ddr_cmds_data[12][0] = 0x60003fce;
467 msg->ddr_cmds_data[12][1] = 0x600011ea;
468 msg->ddr_cmds_data[12][2] = 0x60000008;
469
470 msg->cnoc_cmds_num = 1;
471 msg->cnoc_wait_bitmask = 0x0;
472
473 msg->cnoc_cmds_addrs[0] = 0x50054;
474
475 msg->cnoc_cmds_data[0][0] = 0x40000000;
476 }
477
a640_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)478 static void a640_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
479 {
480 /*
481 * Send a single "off" entry just to get things running
482 * TODO: bus scaling
483 */
484 msg->bw_level_num = 1;
485
486 msg->ddr_cmds_num = 3;
487 msg->ddr_wait_bitmask = 0x01;
488
489 msg->ddr_cmds_addrs[0] = 0x50000;
490 msg->ddr_cmds_addrs[1] = 0x5003c;
491 msg->ddr_cmds_addrs[2] = 0x5000c;
492
493 msg->ddr_cmds_data[0][0] = 0x40000000;
494 msg->ddr_cmds_data[0][1] = 0x40000000;
495 msg->ddr_cmds_data[0][2] = 0x40000000;
496
497 /*
498 * These are the CX (CNOC) votes - these are used by the GMU but the
499 * votes are known and fixed for the target
500 */
501 msg->cnoc_cmds_num = 3;
502 msg->cnoc_wait_bitmask = 0x01;
503
504 msg->cnoc_cmds_addrs[0] = 0x50034;
505 msg->cnoc_cmds_addrs[1] = 0x5007c;
506 msg->cnoc_cmds_addrs[2] = 0x5004c;
507
508 msg->cnoc_cmds_data[0][0] = 0x40000000;
509 msg->cnoc_cmds_data[0][1] = 0x00000000;
510 msg->cnoc_cmds_data[0][2] = 0x40000000;
511
512 msg->cnoc_cmds_data[1][0] = 0x60000001;
513 msg->cnoc_cmds_data[1][1] = 0x20000001;
514 msg->cnoc_cmds_data[1][2] = 0x60000001;
515 }
516
a650_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)517 static void a650_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
518 {
519 /*
520 * Send a single "off" entry just to get things running
521 * TODO: bus scaling
522 */
523 msg->bw_level_num = 1;
524
525 msg->ddr_cmds_num = 3;
526 msg->ddr_wait_bitmask = 0x01;
527
528 msg->ddr_cmds_addrs[0] = 0x50000;
529 msg->ddr_cmds_addrs[1] = 0x50004;
530 msg->ddr_cmds_addrs[2] = 0x5007c;
531
532 msg->ddr_cmds_data[0][0] = 0x40000000;
533 msg->ddr_cmds_data[0][1] = 0x40000000;
534 msg->ddr_cmds_data[0][2] = 0x40000000;
535
536 /*
537 * These are the CX (CNOC) votes - these are used by the GMU but the
538 * votes are known and fixed for the target
539 */
540 msg->cnoc_cmds_num = 1;
541 msg->cnoc_wait_bitmask = 0x01;
542
543 msg->cnoc_cmds_addrs[0] = 0x500a4;
544 msg->cnoc_cmds_data[0][0] = 0x40000000;
545 msg->cnoc_cmds_data[1][0] = 0x60000001;
546 }
547
a690_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)548 static void a690_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
549 {
550 /*
551 * Send a single "off" entry just to get things running
552 * TODO: bus scaling
553 */
554 msg->bw_level_num = 1;
555
556 msg->ddr_cmds_num = 3;
557 msg->ddr_wait_bitmask = 0x01;
558
559 msg->ddr_cmds_addrs[0] = 0x50004;
560 msg->ddr_cmds_addrs[1] = 0x50000;
561 msg->ddr_cmds_addrs[2] = 0x500ac;
562
563 msg->ddr_cmds_data[0][0] = 0x40000000;
564 msg->ddr_cmds_data[0][1] = 0x40000000;
565 msg->ddr_cmds_data[0][2] = 0x40000000;
566
567 /*
568 * These are the CX (CNOC) votes - these are used by the GMU but the
569 * votes are known and fixed for the target
570 */
571 msg->cnoc_cmds_num = 1;
572 msg->cnoc_wait_bitmask = 0x01;
573
574 msg->cnoc_cmds_addrs[0] = 0x5003c;
575 msg->cnoc_cmds_data[0][0] = 0x40000000;
576 msg->cnoc_cmds_data[1][0] = 0x60000001;
577 }
578
a660_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)579 static void a660_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
580 {
581 /*
582 * Send a single "off" entry just to get things running
583 * TODO: bus scaling
584 */
585 msg->bw_level_num = 1;
586
587 msg->ddr_cmds_num = 3;
588 msg->ddr_wait_bitmask = 0x01;
589
590 msg->ddr_cmds_addrs[0] = 0x50004;
591 msg->ddr_cmds_addrs[1] = 0x500a0;
592 msg->ddr_cmds_addrs[2] = 0x50000;
593
594 msg->ddr_cmds_data[0][0] = 0x40000000;
595 msg->ddr_cmds_data[0][1] = 0x40000000;
596 msg->ddr_cmds_data[0][2] = 0x40000000;
597
598 /*
599 * These are the CX (CNOC) votes - these are used by the GMU but the
600 * votes are known and fixed for the target
601 */
602 msg->cnoc_cmds_num = 1;
603 msg->cnoc_wait_bitmask = 0x01;
604
605 msg->cnoc_cmds_addrs[0] = 0x50070;
606 msg->cnoc_cmds_data[0][0] = 0x40000000;
607 msg->cnoc_cmds_data[1][0] = 0x60000001;
608 }
609
a663_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)610 static void a663_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
611 {
612 /*
613 * Send a single "off" entry just to get things running
614 * TODO: bus scaling
615 */
616 msg->bw_level_num = 1;
617
618 msg->ddr_cmds_num = 3;
619 msg->ddr_wait_bitmask = 0x07;
620
621 msg->ddr_cmds_addrs[0] = 0x50004;
622 msg->ddr_cmds_addrs[1] = 0x50000;
623 msg->ddr_cmds_addrs[2] = 0x500b4;
624
625 msg->ddr_cmds_data[0][0] = 0x40000000;
626 msg->ddr_cmds_data[0][1] = 0x40000000;
627 msg->ddr_cmds_data[0][2] = 0x40000000;
628
629 /*
630 * These are the CX (CNOC) votes - these are used by the GMU but the
631 * votes are known and fixed for the target
632 */
633 msg->cnoc_cmds_num = 1;
634 msg->cnoc_wait_bitmask = 0x01;
635
636 msg->cnoc_cmds_addrs[0] = 0x50058;
637 msg->cnoc_cmds_data[0][0] = 0x40000000;
638 msg->cnoc_cmds_data[1][0] = 0x60000001;
639 }
640
adreno_7c3_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)641 static void adreno_7c3_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
642 {
643 /*
644 * Send a single "off" entry just to get things running
645 * TODO: bus scaling
646 */
647 msg->bw_level_num = 1;
648
649 msg->ddr_cmds_num = 3;
650 msg->ddr_wait_bitmask = 0x07;
651
652 msg->ddr_cmds_addrs[0] = 0x50004;
653 msg->ddr_cmds_addrs[1] = 0x50000;
654 msg->ddr_cmds_addrs[2] = 0x50088;
655
656 msg->ddr_cmds_data[0][0] = 0x40000000;
657 msg->ddr_cmds_data[0][1] = 0x40000000;
658 msg->ddr_cmds_data[0][2] = 0x40000000;
659
660 /*
661 * These are the CX (CNOC) votes - these are used by the GMU but the
662 * votes are known and fixed for the target
663 */
664 msg->cnoc_cmds_num = 1;
665 msg->cnoc_wait_bitmask = 0x01;
666
667 msg->cnoc_cmds_addrs[0] = 0x5006c;
668 msg->cnoc_cmds_data[0][0] = 0x40000000;
669 msg->cnoc_cmds_data[1][0] = 0x60000001;
670 }
671
a730_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)672 static void a730_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
673 {
674 msg->bw_level_num = 12;
675
676 msg->ddr_cmds_num = 3;
677 msg->ddr_wait_bitmask = 0x7;
678
679 msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0");
680 msg->ddr_cmds_addrs[1] = cmd_db_read_addr("MC0");
681 msg->ddr_cmds_addrs[2] = cmd_db_read_addr("ACV");
682
683 msg->ddr_cmds_data[0][0] = 0x40000000;
684 msg->ddr_cmds_data[0][1] = 0x40000000;
685 msg->ddr_cmds_data[0][2] = 0x40000000;
686 msg->ddr_cmds_data[1][0] = 0x600002e8;
687 msg->ddr_cmds_data[1][1] = 0x600003d0;
688 msg->ddr_cmds_data[1][2] = 0x60000008;
689 msg->ddr_cmds_data[2][0] = 0x6000068d;
690 msg->ddr_cmds_data[2][1] = 0x6000089a;
691 msg->ddr_cmds_data[2][2] = 0x60000008;
692 msg->ddr_cmds_data[3][0] = 0x600007f2;
693 msg->ddr_cmds_data[3][1] = 0x60000a6e;
694 msg->ddr_cmds_data[3][2] = 0x60000008;
695 msg->ddr_cmds_data[4][0] = 0x600009e5;
696 msg->ddr_cmds_data[4][1] = 0x60000cfd;
697 msg->ddr_cmds_data[4][2] = 0x60000008;
698 msg->ddr_cmds_data[5][0] = 0x60000b29;
699 msg->ddr_cmds_data[5][1] = 0x60000ea6;
700 msg->ddr_cmds_data[5][2] = 0x60000008;
701 msg->ddr_cmds_data[6][0] = 0x60001698;
702 msg->ddr_cmds_data[6][1] = 0x60001da8;
703 msg->ddr_cmds_data[6][2] = 0x60000008;
704 msg->ddr_cmds_data[7][0] = 0x600018d2;
705 msg->ddr_cmds_data[7][1] = 0x60002093;
706 msg->ddr_cmds_data[7][2] = 0x60000008;
707 msg->ddr_cmds_data[8][0] = 0x60001e66;
708 msg->ddr_cmds_data[8][1] = 0x600027e6;
709 msg->ddr_cmds_data[8][2] = 0x60000008;
710 msg->ddr_cmds_data[9][0] = 0x600027c2;
711 msg->ddr_cmds_data[9][1] = 0x6000342f;
712 msg->ddr_cmds_data[9][2] = 0x60000008;
713 msg->ddr_cmds_data[10][0] = 0x60002e71;
714 msg->ddr_cmds_data[10][1] = 0x60003cf5;
715 msg->ddr_cmds_data[10][2] = 0x60000008;
716 msg->ddr_cmds_data[11][0] = 0x600030ae;
717 msg->ddr_cmds_data[11][1] = 0x60003fe5;
718 msg->ddr_cmds_data[11][2] = 0x60000008;
719
720 msg->cnoc_cmds_num = 1;
721 msg->cnoc_wait_bitmask = 0x1;
722
723 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");
724 msg->cnoc_cmds_data[0][0] = 0x40000000;
725 msg->cnoc_cmds_data[1][0] = 0x60000001;
726 }
727
a740_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)728 static void a740_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
729 {
730 msg->bw_level_num = 1;
731
732 msg->ddr_cmds_num = 3;
733 msg->ddr_wait_bitmask = 0x7;
734
735 msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0");
736 msg->ddr_cmds_addrs[1] = cmd_db_read_addr("MC0");
737 msg->ddr_cmds_addrs[2] = cmd_db_read_addr("ACV");
738
739 msg->ddr_cmds_data[0][0] = 0x40000000;
740 msg->ddr_cmds_data[0][1] = 0x40000000;
741 msg->ddr_cmds_data[0][2] = 0x40000000;
742
743 /* TODO: add a proper dvfs table */
744
745 msg->cnoc_cmds_num = 1;
746 msg->cnoc_wait_bitmask = 0x1;
747
748 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0");
749 msg->cnoc_cmds_data[0][0] = 0x40000000;
750 msg->cnoc_cmds_data[1][0] = 0x60000001;
751 }
752
a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table * msg)753 static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg)
754 {
755 /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */
756 msg->bw_level_num = 1;
757
758 msg->ddr_cmds_num = 3;
759 msg->ddr_wait_bitmask = 0x07;
760
761 msg->ddr_cmds_addrs[0] = 0x50000;
762 msg->ddr_cmds_addrs[1] = 0x5005c;
763 msg->ddr_cmds_addrs[2] = 0x5000c;
764
765 msg->ddr_cmds_data[0][0] = 0x40000000;
766 msg->ddr_cmds_data[0][1] = 0x40000000;
767 msg->ddr_cmds_data[0][2] = 0x40000000;
768
769 /*
770 * These are the CX (CNOC) votes. This is used but the values for the
771 * sdm845 GMU are known and fixed so we can hard code them.
772 */
773
774 msg->cnoc_cmds_num = 3;
775 msg->cnoc_wait_bitmask = 0x05;
776
777 msg->cnoc_cmds_addrs[0] = 0x50034;
778 msg->cnoc_cmds_addrs[1] = 0x5007c;
779 msg->cnoc_cmds_addrs[2] = 0x5004c;
780
781 msg->cnoc_cmds_data[0][0] = 0x40000000;
782 msg->cnoc_cmds_data[0][1] = 0x00000000;
783 msg->cnoc_cmds_data[0][2] = 0x40000000;
784
785 msg->cnoc_cmds_data[1][0] = 0x60000001;
786 msg->cnoc_cmds_data[1][1] = 0x20000001;
787 msg->cnoc_cmds_data[1][2] = 0x60000001;
788 }
789
790
a6xx_hfi_send_bw_table(struct a6xx_gmu * gmu)791 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu)
792 {
793 struct a6xx_hfi_msg_bw_table *msg;
794 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu);
795 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base;
796 const struct a6xx_info *info = adreno_gpu->info->a6xx;
797
798 if (gmu->bw_table)
799 goto send;
800
801 msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL);
802 if (!msg)
803 return -ENOMEM;
804
805 if (info->bcms && gmu->nr_gpu_bws > 1)
806 a6xx_generate_bw_table(info, gmu, msg);
807 else if (adreno_is_a618(adreno_gpu))
808 a618_build_bw_table(msg);
809 else if (adreno_is_a619(adreno_gpu))
810 a619_build_bw_table(msg);
811 else if (adreno_is_a640_family(adreno_gpu))
812 a640_build_bw_table(msg);
813 else if (adreno_is_a650(adreno_gpu))
814 a650_build_bw_table(msg);
815 else if (adreno_is_7c3(adreno_gpu))
816 adreno_7c3_build_bw_table(msg);
817 else if (adreno_is_a660(adreno_gpu))
818 a660_build_bw_table(msg);
819 else if (adreno_is_a663(adreno_gpu))
820 a663_build_bw_table(msg);
821 else if (adreno_is_a690(adreno_gpu))
822 a690_build_bw_table(msg);
823 else if (adreno_is_a730(adreno_gpu))
824 a730_build_bw_table(msg);
825 else if (adreno_is_a740_family(adreno_gpu))
826 a740_build_bw_table(msg);
827 else
828 a6xx_build_bw_table(msg);
829
830 gmu->bw_table = msg;
831
832 send:
833 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)),
834 NULL, 0);
835 }
836
a6xx_hfi_feature_ctrl_msg(struct a6xx_gmu * gmu,u32 feature,u32 enable,u32 data)837 static int a6xx_hfi_feature_ctrl_msg(struct a6xx_gmu *gmu, u32 feature, u32 enable, u32 data)
838 {
839 struct a6xx_hfi_msg_feature_ctrl msg = {
840 .feature = feature,
841 .enable = enable,
842 .data = data,
843 };
844
845 return a6xx_hfi_send_msg(gmu, HFI_H2F_FEATURE_CTRL, &msg, sizeof(msg), NULL, 0);
846 }
847
848 #define HFI_FEATURE_IFPC 9
849 #define IFPC_LONG_HYST 0x1680
850
a6xx_hfi_enable_ifpc(struct a6xx_gmu * gmu)851 static int a6xx_hfi_enable_ifpc(struct a6xx_gmu *gmu)
852 {
853 if (gmu->idle_level != GMU_IDLE_STATE_IFPC)
854 return 0;
855
856 return a6xx_hfi_feature_ctrl_msg(gmu, HFI_FEATURE_IFPC, 1, IFPC_LONG_HYST);
857 }
858
859 #define HFI_FEATURE_ACD 12
860
a6xx_hfi_enable_acd(struct a6xx_gmu * gmu)861 static int a6xx_hfi_enable_acd(struct a6xx_gmu *gmu)
862 {
863 struct a6xx_hfi_acd_table *acd_table = &gmu->acd_table;
864 int ret;
865
866 if (!acd_table->enable_by_level)
867 return 0;
868
869 /* Enable ACD feature at GMU */
870 ret = a6xx_hfi_feature_ctrl_msg(gmu, HFI_FEATURE_ACD, 1, 0);
871 if (ret) {
872 DRM_DEV_ERROR(gmu->dev, "Unable to enable ACD (%d)\n", ret);
873 return ret;
874 }
875
876 /* Send ACD table to GMU */
877 ret = a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_ACD, acd_table, sizeof(*acd_table), NULL, 0);
878 if (ret) {
879 DRM_DEV_ERROR(gmu->dev, "Unable to ACD table (%d)\n", ret);
880 return ret;
881 }
882
883 return 0;
884 }
885
a6xx_hfi_send_test(struct a6xx_gmu * gmu)886 static int a6xx_hfi_send_test(struct a6xx_gmu *gmu)
887 {
888 struct a6xx_hfi_msg_test msg = { 0 };
889
890 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg),
891 NULL, 0);
892 }
893
a6xx_hfi_send_start(struct a6xx_gmu * gmu)894 static int a6xx_hfi_send_start(struct a6xx_gmu *gmu)
895 {
896 struct a6xx_hfi_msg_start msg = { 0 };
897
898 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_START, &msg, sizeof(msg),
899 NULL, 0);
900 }
901
a6xx_hfi_send_core_fw_start(struct a6xx_gmu * gmu)902 static int a6xx_hfi_send_core_fw_start(struct a6xx_gmu *gmu)
903 {
904 struct a6xx_hfi_msg_core_fw_start msg = { 0 };
905
906 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_CORE_FW_START, &msg,
907 sizeof(msg), NULL, 0);
908 }
909
a6xx_hfi_set_freq(struct a6xx_gmu * gmu,u32 freq_index,u32 bw_index)910 int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, u32 freq_index, u32 bw_index)
911 {
912 struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 };
913
914 msg.ack_type = 1; /* blocking */
915 msg.freq = freq_index;
916 msg.bw = bw_index;
917
918 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_GX_BW_PERF_VOTE, &msg,
919 sizeof(msg), NULL, 0);
920 }
921
a6xx_hfi_send_prep_slumber(struct a6xx_gmu * gmu)922 int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu)
923 {
924 struct a6xx_hfi_prep_slumber_cmd msg = { 0 };
925
926 /* TODO: should freq and bw fields be non-zero ? */
927
928 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PREPARE_SLUMBER, &msg,
929 sizeof(msg), NULL, 0);
930 }
931
a6xx_hfi_start_v1(struct a6xx_gmu * gmu,int boot_state)932 static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu, int boot_state)
933 {
934 int ret;
935
936 ret = a6xx_hfi_send_gmu_init(gmu, boot_state);
937 if (ret)
938 return ret;
939
940 ret = a6xx_hfi_get_fw_version(gmu, NULL);
941 if (ret)
942 return ret;
943
944 /*
945 * We have to get exchange version numbers per the sequence but at this
946 * point th kernel driver doesn't need to know the exact version of
947 * the GMU firmware
948 */
949
950 ret = a6xx_hfi_send_perf_table_v1(gmu);
951 if (ret)
952 return ret;
953
954 ret = a6xx_hfi_send_bw_table(gmu);
955 if (ret)
956 return ret;
957
958 /*
959 * Let the GMU know that there won't be any more HFI messages until next
960 * boot
961 */
962 a6xx_hfi_send_test(gmu);
963
964 return 0;
965 }
966
a6xx_hfi_start(struct a6xx_gmu * gmu,int boot_state)967 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state)
968 {
969 int ret;
970
971 if (gmu->legacy)
972 return a6xx_hfi_start_v1(gmu, boot_state);
973
974
975 ret = a6xx_hfi_send_perf_table(gmu);
976 if (ret)
977 return ret;
978
979 ret = a6xx_hfi_send_bw_table(gmu);
980 if (ret)
981 return ret;
982
983 ret = a6xx_hfi_enable_acd(gmu);
984 if (ret)
985 return ret;
986
987 ret = a6xx_hfi_enable_ifpc(gmu);
988 if (ret)
989 return ret;
990
991 ret = a6xx_hfi_send_core_fw_start(gmu);
992 if (ret)
993 return ret;
994
995 /*
996 * Downstream driver sends this in its "a6xx_hw_init" equivalent,
997 * but seems to be no harm in sending it here
998 */
999 ret = a6xx_hfi_send_start(gmu);
1000 if (ret)
1001 return ret;
1002
1003 return 0;
1004 }
1005
a6xx_hfi_stop(struct a6xx_gmu * gmu)1006 void a6xx_hfi_stop(struct a6xx_gmu *gmu)
1007 {
1008 int i;
1009
1010 for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) {
1011 struct a6xx_hfi_queue *queue = &gmu->queues[i];
1012
1013 if (!queue->header)
1014 continue;
1015
1016 if (queue->header->read_index != queue->header->write_index)
1017 DRM_DEV_ERROR(gmu->dev, "HFI queue %d is not empty\n", i);
1018
1019 queue->header->read_index = 0;
1020 queue->header->write_index = 0;
1021
1022 memset(&queue->history, 0xff, sizeof(queue->history));
1023 queue->history_idx = 0;
1024 }
1025 }
1026
a6xx_hfi_queue_init(struct a6xx_hfi_queue * queue,struct a6xx_hfi_queue_header * header,void * virt,u64 iova,u32 id)1027 static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue,
1028 struct a6xx_hfi_queue_header *header, void *virt, u64 iova,
1029 u32 id)
1030 {
1031 spin_lock_init(&queue->lock);
1032 queue->header = header;
1033 queue->data = virt;
1034 atomic_set(&queue->seqnum, 0);
1035
1036 memset(&queue->history, 0xff, sizeof(queue->history));
1037 queue->history_idx = 0;
1038
1039 /* Set up the shared memory header */
1040 header->iova = iova;
1041 header->type = 10 << 8 | id;
1042 header->status = 1;
1043 header->size = SZ_4K >> 2;
1044 header->msg_size = 0;
1045 header->dropped = 0;
1046 header->rx_watermark = 1;
1047 header->tx_watermark = 1;
1048 header->rx_request = 1;
1049 header->tx_request = 0;
1050 header->read_index = 0;
1051 header->write_index = 0;
1052 }
1053
a6xx_hfi_init(struct a6xx_gmu * gmu)1054 void a6xx_hfi_init(struct a6xx_gmu *gmu)
1055 {
1056 struct a6xx_gmu_bo *hfi = &gmu->hfi;
1057 struct a6xx_hfi_queue_table_header *table = hfi->virt;
1058 struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table);
1059 u64 offset;
1060 int table_size;
1061
1062 /*
1063 * The table size is the size of the table header plus all of the queue
1064 * headers
1065 */
1066 table_size = sizeof(*table);
1067 table_size += (ARRAY_SIZE(gmu->queues) *
1068 sizeof(struct a6xx_hfi_queue_header));
1069
1070 table->version = 0;
1071 table->size = table_size;
1072 /* First queue header is located immediately after the table header */
1073 table->qhdr0_offset = sizeof(*table) >> 2;
1074 table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2;
1075 table->num_queues = ARRAY_SIZE(gmu->queues);
1076 table->active_queues = ARRAY_SIZE(gmu->queues);
1077
1078 /* Command queue */
1079 offset = SZ_4K;
1080 a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset,
1081 hfi->iova + offset, 0);
1082
1083 /* GMU response queue */
1084 offset += SZ_4K;
1085 a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset,
1086 hfi->iova + offset, gmu->legacy ? 4 : 1);
1087 }
1088