1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */ 3 4 #include <linux/completion.h> 5 #include <linux/circ_buf.h> 6 #include <linux/list.h> 7 8 #include <soc/qcom/cmd-db.h> 9 #include <soc/qcom/tcs.h> 10 11 #include "a6xx_gmu.h" 12 #include "a6xx_gmu.xml.h" 13 #include "a6xx_gpu.h" 14 15 #define HFI_MSG_ID(val) [val] = #val 16 17 static const char * const a6xx_hfi_msg_id[] = { 18 HFI_MSG_ID(HFI_H2F_MSG_INIT), 19 HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION), 20 HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE), 21 HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE), 22 HFI_MSG_ID(HFI_H2F_MSG_TEST), 23 HFI_MSG_ID(HFI_H2F_MSG_START), 24 HFI_MSG_ID(HFI_H2F_FEATURE_CTRL), 25 HFI_MSG_ID(HFI_H2F_MSG_CORE_FW_START), 26 HFI_MSG_ID(HFI_H2F_MSG_GX_BW_PERF_VOTE), 27 HFI_MSG_ID(HFI_H2F_MSG_PREPARE_SLUMBER), 28 }; 29 30 static int a6xx_hfi_queue_read(struct a6xx_gmu *gmu, 31 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords) 32 { 33 struct a6xx_hfi_queue_header *header = queue->header; 34 u32 i, hdr, index = header->read_index; 35 36 if (header->read_index == header->write_index) { 37 header->rx_request = 1; 38 return 0; 39 } 40 41 hdr = queue->data[index]; 42 43 queue->history[(queue->history_idx++) % HFI_HISTORY_SZ] = index; 44 45 /* 46 * If we are to assume that the GMU firmware is in fact a rational actor 47 * and is programmed to not send us a larger response than we expect 48 * then we can also assume that if the header size is unexpectedly large 49 * that it is due to memory corruption and/or hardware failure. In this 50 * case the only reasonable course of action is to BUG() to help harden 51 * the failure. 52 */ 53 54 BUG_ON(HFI_HEADER_SIZE(hdr) > dwords); 55 56 for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) { 57 data[i] = queue->data[index]; 58 index = (index + 1) % header->size; 59 } 60 61 if (!gmu->legacy) 62 index = ALIGN(index, 4) % header->size; 63 64 header->read_index = index; 65 return HFI_HEADER_SIZE(hdr); 66 } 67 68 static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu, 69 struct a6xx_hfi_queue *queue, u32 *data, u32 dwords) 70 { 71 struct a6xx_hfi_queue_header *header = queue->header; 72 u32 i, space, index = header->write_index; 73 74 spin_lock(&queue->lock); 75 76 space = CIRC_SPACE(header->write_index, header->read_index, 77 header->size); 78 if (space < dwords) { 79 header->dropped++; 80 spin_unlock(&queue->lock); 81 return -ENOSPC; 82 } 83 84 queue->history[(queue->history_idx++) % HFI_HISTORY_SZ] = index; 85 86 for (i = 0; i < dwords; i++) { 87 queue->data[index] = data[i]; 88 index = (index + 1) % header->size; 89 } 90 91 /* Cookify any non used data at the end of the write buffer */ 92 if (!gmu->legacy) { 93 for (; index % 4; index = (index + 1) % header->size) 94 queue->data[index] = 0xfafafafa; 95 } 96 97 header->write_index = index; 98 spin_unlock(&queue->lock); 99 100 gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01); 101 return 0; 102 } 103 104 static int a6xx_hfi_wait_for_msg_interrupt(struct a6xx_gmu *gmu, u32 id, u32 seqnum) 105 { 106 int ret; 107 u32 val; 108 109 /* Wait for a response */ 110 ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val, 111 val & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 100, 1000000); 112 113 if (ret) { 114 DRM_DEV_ERROR(gmu->dev, 115 "Message %s id %d timed out waiting for response\n", 116 a6xx_hfi_msg_id[id], seqnum); 117 return -ETIMEDOUT; 118 } 119 120 /* Clear the interrupt */ 121 gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 122 A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ); 123 124 return 0; 125 } 126 127 static int a6xx_hfi_wait_for_ack(struct a6xx_gmu *gmu, u32 id, u32 seqnum, 128 u32 *payload, u32 payload_size) 129 { 130 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE]; 131 int ret; 132 133 ret = a6xx_hfi_wait_for_msg_interrupt(gmu, id, seqnum); 134 if (ret) 135 return ret; 136 137 for (;;) { 138 struct a6xx_hfi_msg_response resp; 139 140 /* Get the next packet */ 141 ret = a6xx_hfi_queue_read(gmu, queue, (u32 *) &resp, 142 sizeof(resp) >> 2); 143 144 /* If the queue is empty, there may have been previous missed 145 * responses that preceded the response to our packet. Wait 146 * further before we give up. 147 */ 148 if (!ret) { 149 ret = a6xx_hfi_wait_for_msg_interrupt(gmu, id, seqnum); 150 if (ret) { 151 DRM_DEV_ERROR(gmu->dev, 152 "The HFI response queue is unexpectedly empty\n"); 153 return ret; 154 } 155 continue; 156 } 157 158 if (HFI_HEADER_ID(resp.header) == HFI_F2H_MSG_ERROR) { 159 struct a6xx_hfi_msg_error *error = 160 (struct a6xx_hfi_msg_error *) &resp; 161 162 DRM_DEV_ERROR(gmu->dev, "GMU firmware error %d\n", 163 error->code); 164 continue; 165 } 166 167 if (seqnum != HFI_HEADER_SEQNUM(resp.ret_header)) { 168 DRM_DEV_ERROR(gmu->dev, 169 "Unexpected message id %d on the response queue\n", 170 HFI_HEADER_SEQNUM(resp.ret_header)); 171 continue; 172 } 173 174 if (resp.error) { 175 DRM_DEV_ERROR(gmu->dev, 176 "Message %s id %d returned error %d\n", 177 a6xx_hfi_msg_id[id], seqnum, resp.error); 178 return -EINVAL; 179 } 180 181 /* All is well, copy over the buffer */ 182 if (payload && payload_size) 183 memcpy(payload, resp.payload, 184 min_t(u32, payload_size, sizeof(resp.payload))); 185 186 return 0; 187 } 188 } 189 190 static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id, 191 void *data, u32 size, u32 *payload, u32 payload_size) 192 { 193 struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE]; 194 int ret, dwords = size >> 2; 195 u32 seqnum; 196 197 seqnum = atomic_inc_return(&queue->seqnum) % 0xfff; 198 199 /* First dword of the message is the message header - fill it in */ 200 *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) | 201 (dwords << 8) | id; 202 203 ret = a6xx_hfi_queue_write(gmu, queue, data, dwords); 204 if (ret) { 205 DRM_DEV_ERROR(gmu->dev, "Unable to send message %s id %d\n", 206 a6xx_hfi_msg_id[id], seqnum); 207 return ret; 208 } 209 210 return a6xx_hfi_wait_for_ack(gmu, id, seqnum, payload, payload_size); 211 } 212 213 static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state) 214 { 215 struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 }; 216 217 msg.dbg_buffer_addr = (u32) gmu->debug.iova; 218 msg.dbg_buffer_size = (u32) gmu->debug.size; 219 msg.boot_state = boot_state; 220 221 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg), 222 NULL, 0); 223 } 224 225 static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version) 226 { 227 struct a6xx_hfi_msg_fw_version msg = { 0 }; 228 229 /* Currently supporting version 1.10 */ 230 msg.supported_version = (1 << 28) | (1 << 19) | (1 << 17); 231 232 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg), 233 version, sizeof(*version)); 234 } 235 236 static int a6xx_hfi_send_perf_table_v1(struct a6xx_gmu *gmu) 237 { 238 struct a6xx_hfi_msg_perf_table_v1 msg = { 0 }; 239 int i; 240 241 msg.num_gpu_levels = gmu->nr_gpu_freqs; 242 msg.num_gmu_levels = gmu->nr_gmu_freqs; 243 244 for (i = 0; i < gmu->nr_gpu_freqs; i++) { 245 msg.gx_votes[i].vote = gmu->gx_arc_votes[i]; 246 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000; 247 } 248 249 for (i = 0; i < gmu->nr_gmu_freqs; i++) { 250 msg.cx_votes[i].vote = gmu->cx_arc_votes[i]; 251 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000; 252 } 253 254 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg), 255 NULL, 0); 256 } 257 258 static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu) 259 { 260 struct a6xx_hfi_msg_perf_table msg = { 0 }; 261 int i; 262 263 msg.num_gpu_levels = gmu->nr_gpu_freqs; 264 msg.num_gmu_levels = gmu->nr_gmu_freqs; 265 266 for (i = 0; i < gmu->nr_gpu_freqs; i++) { 267 msg.gx_votes[i].vote = gmu->gx_arc_votes[i]; 268 msg.gx_votes[i].acd = 0xffffffff; 269 msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000; 270 } 271 272 for (i = 0; i < gmu->nr_gmu_freqs; i++) { 273 msg.cx_votes[i].vote = gmu->cx_arc_votes[i]; 274 msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000; 275 } 276 277 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg), 278 NULL, 0); 279 } 280 281 static void a6xx_generate_bw_table(const struct a6xx_info *info, struct a6xx_gmu *gmu, 282 struct a6xx_hfi_msg_bw_table *msg) 283 { 284 unsigned int i, j; 285 286 for (i = 0; i < GMU_MAX_BCMS; i++) { 287 if (!info->bcms[i].name) 288 break; 289 msg->ddr_cmds_addrs[i] = cmd_db_read_addr(info->bcms[i].name); 290 } 291 msg->ddr_cmds_num = i; 292 293 for (i = 0; i < gmu->nr_gpu_bws; ++i) 294 for (j = 0; j < msg->ddr_cmds_num; j++) 295 msg->ddr_cmds_data[i][j] = gmu->gpu_ib_votes[i][j]; 296 msg->bw_level_num = gmu->nr_gpu_bws; 297 298 /* Compute the wait bitmask with each BCM having the commit bit */ 299 msg->ddr_wait_bitmask = 0; 300 for (j = 0; j < msg->ddr_cmds_num; j++) 301 if (msg->ddr_cmds_data[0][j] & BCM_TCS_CMD_COMMIT_MASK) 302 msg->ddr_wait_bitmask |= BIT(j); 303 304 /* 305 * These are the CX (CNOC) votes - these are used by the GMU 306 * The 'CN0' BCM is used on all targets, and votes are basically 307 * 'off' and 'on' states with first bit to enable the path. 308 */ 309 310 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0"); 311 msg->cnoc_cmds_num = 1; 312 313 msg->cnoc_cmds_data[0][0] = BCM_TCS_CMD(true, false, 0, 0); 314 msg->cnoc_cmds_data[1][0] = BCM_TCS_CMD(true, true, 0, BIT(0)); 315 316 /* Compute the wait bitmask with each BCM having the commit bit */ 317 msg->cnoc_wait_bitmask = 0; 318 for (j = 0; j < msg->cnoc_cmds_num; j++) 319 if (msg->cnoc_cmds_data[0][j] & BCM_TCS_CMD_COMMIT_MASK) 320 msg->cnoc_wait_bitmask |= BIT(j); 321 } 322 323 static void a618_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 324 { 325 /* Send a single "off" entry since the 618 GMU doesn't do bus scaling */ 326 msg->bw_level_num = 1; 327 328 msg->ddr_cmds_num = 3; 329 msg->ddr_wait_bitmask = 0x01; 330 331 msg->ddr_cmds_addrs[0] = 0x50000; 332 msg->ddr_cmds_addrs[1] = 0x5003c; 333 msg->ddr_cmds_addrs[2] = 0x5000c; 334 335 msg->ddr_cmds_data[0][0] = 0x40000000; 336 msg->ddr_cmds_data[0][1] = 0x40000000; 337 msg->ddr_cmds_data[0][2] = 0x40000000; 338 339 /* 340 * These are the CX (CNOC) votes - these are used by the GMU but the 341 * votes are known and fixed for the target 342 */ 343 msg->cnoc_cmds_num = 1; 344 msg->cnoc_wait_bitmask = 0x01; 345 346 msg->cnoc_cmds_addrs[0] = 0x5007c; 347 msg->cnoc_cmds_data[0][0] = 0x40000000; 348 msg->cnoc_cmds_data[1][0] = 0x60000001; 349 } 350 351 static void a619_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 352 { 353 msg->bw_level_num = 13; 354 355 msg->ddr_cmds_num = 3; 356 msg->ddr_wait_bitmask = 0x0; 357 358 msg->ddr_cmds_addrs[0] = 0x50000; 359 msg->ddr_cmds_addrs[1] = 0x50004; 360 msg->ddr_cmds_addrs[2] = 0x50080; 361 362 msg->ddr_cmds_data[0][0] = 0x40000000; 363 msg->ddr_cmds_data[0][1] = 0x40000000; 364 msg->ddr_cmds_data[0][2] = 0x40000000; 365 msg->ddr_cmds_data[1][0] = 0x6000030c; 366 msg->ddr_cmds_data[1][1] = 0x600000db; 367 msg->ddr_cmds_data[1][2] = 0x60000008; 368 msg->ddr_cmds_data[2][0] = 0x60000618; 369 msg->ddr_cmds_data[2][1] = 0x600001b6; 370 msg->ddr_cmds_data[2][2] = 0x60000008; 371 msg->ddr_cmds_data[3][0] = 0x60000925; 372 msg->ddr_cmds_data[3][1] = 0x60000291; 373 msg->ddr_cmds_data[3][2] = 0x60000008; 374 msg->ddr_cmds_data[4][0] = 0x60000dc1; 375 msg->ddr_cmds_data[4][1] = 0x600003dc; 376 msg->ddr_cmds_data[4][2] = 0x60000008; 377 msg->ddr_cmds_data[5][0] = 0x600010ad; 378 msg->ddr_cmds_data[5][1] = 0x600004ae; 379 msg->ddr_cmds_data[5][2] = 0x60000008; 380 msg->ddr_cmds_data[6][0] = 0x600014c3; 381 msg->ddr_cmds_data[6][1] = 0x600005d4; 382 msg->ddr_cmds_data[6][2] = 0x60000008; 383 msg->ddr_cmds_data[7][0] = 0x6000176a; 384 msg->ddr_cmds_data[7][1] = 0x60000693; 385 msg->ddr_cmds_data[7][2] = 0x60000008; 386 msg->ddr_cmds_data[8][0] = 0x60001f01; 387 msg->ddr_cmds_data[8][1] = 0x600008b5; 388 msg->ddr_cmds_data[8][2] = 0x60000008; 389 msg->ddr_cmds_data[9][0] = 0x60002940; 390 msg->ddr_cmds_data[9][1] = 0x60000b95; 391 msg->ddr_cmds_data[9][2] = 0x60000008; 392 msg->ddr_cmds_data[10][0] = 0x60002f68; 393 msg->ddr_cmds_data[10][1] = 0x60000d50; 394 msg->ddr_cmds_data[10][2] = 0x60000008; 395 msg->ddr_cmds_data[11][0] = 0x60003700; 396 msg->ddr_cmds_data[11][1] = 0x60000f71; 397 msg->ddr_cmds_data[11][2] = 0x60000008; 398 msg->ddr_cmds_data[12][0] = 0x60003fce; 399 msg->ddr_cmds_data[12][1] = 0x600011ea; 400 msg->ddr_cmds_data[12][2] = 0x60000008; 401 402 msg->cnoc_cmds_num = 1; 403 msg->cnoc_wait_bitmask = 0x0; 404 405 msg->cnoc_cmds_addrs[0] = 0x50054; 406 407 msg->cnoc_cmds_data[0][0] = 0x40000000; 408 } 409 410 static void a640_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 411 { 412 /* 413 * Send a single "off" entry just to get things running 414 * TODO: bus scaling 415 */ 416 msg->bw_level_num = 1; 417 418 msg->ddr_cmds_num = 3; 419 msg->ddr_wait_bitmask = 0x01; 420 421 msg->ddr_cmds_addrs[0] = 0x50000; 422 msg->ddr_cmds_addrs[1] = 0x5003c; 423 msg->ddr_cmds_addrs[2] = 0x5000c; 424 425 msg->ddr_cmds_data[0][0] = 0x40000000; 426 msg->ddr_cmds_data[0][1] = 0x40000000; 427 msg->ddr_cmds_data[0][2] = 0x40000000; 428 429 /* 430 * These are the CX (CNOC) votes - these are used by the GMU but the 431 * votes are known and fixed for the target 432 */ 433 msg->cnoc_cmds_num = 3; 434 msg->cnoc_wait_bitmask = 0x01; 435 436 msg->cnoc_cmds_addrs[0] = 0x50034; 437 msg->cnoc_cmds_addrs[1] = 0x5007c; 438 msg->cnoc_cmds_addrs[2] = 0x5004c; 439 440 msg->cnoc_cmds_data[0][0] = 0x40000000; 441 msg->cnoc_cmds_data[0][1] = 0x00000000; 442 msg->cnoc_cmds_data[0][2] = 0x40000000; 443 444 msg->cnoc_cmds_data[1][0] = 0x60000001; 445 msg->cnoc_cmds_data[1][1] = 0x20000001; 446 msg->cnoc_cmds_data[1][2] = 0x60000001; 447 } 448 449 static void a650_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 450 { 451 /* 452 * Send a single "off" entry just to get things running 453 * TODO: bus scaling 454 */ 455 msg->bw_level_num = 1; 456 457 msg->ddr_cmds_num = 3; 458 msg->ddr_wait_bitmask = 0x01; 459 460 msg->ddr_cmds_addrs[0] = 0x50000; 461 msg->ddr_cmds_addrs[1] = 0x50004; 462 msg->ddr_cmds_addrs[2] = 0x5007c; 463 464 msg->ddr_cmds_data[0][0] = 0x40000000; 465 msg->ddr_cmds_data[0][1] = 0x40000000; 466 msg->ddr_cmds_data[0][2] = 0x40000000; 467 468 /* 469 * These are the CX (CNOC) votes - these are used by the GMU but the 470 * votes are known and fixed for the target 471 */ 472 msg->cnoc_cmds_num = 1; 473 msg->cnoc_wait_bitmask = 0x01; 474 475 msg->cnoc_cmds_addrs[0] = 0x500a4; 476 msg->cnoc_cmds_data[0][0] = 0x40000000; 477 msg->cnoc_cmds_data[1][0] = 0x60000001; 478 } 479 480 static void a690_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 481 { 482 /* 483 * Send a single "off" entry just to get things running 484 * TODO: bus scaling 485 */ 486 msg->bw_level_num = 1; 487 488 msg->ddr_cmds_num = 3; 489 msg->ddr_wait_bitmask = 0x01; 490 491 msg->ddr_cmds_addrs[0] = 0x50004; 492 msg->ddr_cmds_addrs[1] = 0x50000; 493 msg->ddr_cmds_addrs[2] = 0x500ac; 494 495 msg->ddr_cmds_data[0][0] = 0x40000000; 496 msg->ddr_cmds_data[0][1] = 0x40000000; 497 msg->ddr_cmds_data[0][2] = 0x40000000; 498 499 /* 500 * These are the CX (CNOC) votes - these are used by the GMU but the 501 * votes are known and fixed for the target 502 */ 503 msg->cnoc_cmds_num = 1; 504 msg->cnoc_wait_bitmask = 0x01; 505 506 msg->cnoc_cmds_addrs[0] = 0x5003c; 507 msg->cnoc_cmds_data[0][0] = 0x40000000; 508 msg->cnoc_cmds_data[1][0] = 0x60000001; 509 } 510 511 static void a660_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 512 { 513 /* 514 * Send a single "off" entry just to get things running 515 * TODO: bus scaling 516 */ 517 msg->bw_level_num = 1; 518 519 msg->ddr_cmds_num = 3; 520 msg->ddr_wait_bitmask = 0x01; 521 522 msg->ddr_cmds_addrs[0] = 0x50004; 523 msg->ddr_cmds_addrs[1] = 0x500a0; 524 msg->ddr_cmds_addrs[2] = 0x50000; 525 526 msg->ddr_cmds_data[0][0] = 0x40000000; 527 msg->ddr_cmds_data[0][1] = 0x40000000; 528 msg->ddr_cmds_data[0][2] = 0x40000000; 529 530 /* 531 * These are the CX (CNOC) votes - these are used by the GMU but the 532 * votes are known and fixed for the target 533 */ 534 msg->cnoc_cmds_num = 1; 535 msg->cnoc_wait_bitmask = 0x01; 536 537 msg->cnoc_cmds_addrs[0] = 0x50070; 538 msg->cnoc_cmds_data[0][0] = 0x40000000; 539 msg->cnoc_cmds_data[1][0] = 0x60000001; 540 } 541 542 static void a663_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 543 { 544 /* 545 * Send a single "off" entry just to get things running 546 * TODO: bus scaling 547 */ 548 msg->bw_level_num = 1; 549 550 msg->ddr_cmds_num = 3; 551 msg->ddr_wait_bitmask = 0x07; 552 553 msg->ddr_cmds_addrs[0] = 0x50004; 554 msg->ddr_cmds_addrs[1] = 0x50000; 555 msg->ddr_cmds_addrs[2] = 0x500b4; 556 557 msg->ddr_cmds_data[0][0] = 0x40000000; 558 msg->ddr_cmds_data[0][1] = 0x40000000; 559 msg->ddr_cmds_data[0][2] = 0x40000000; 560 561 /* 562 * These are the CX (CNOC) votes - these are used by the GMU but the 563 * votes are known and fixed for the target 564 */ 565 msg->cnoc_cmds_num = 1; 566 msg->cnoc_wait_bitmask = 0x01; 567 568 msg->cnoc_cmds_addrs[0] = 0x50058; 569 msg->cnoc_cmds_data[0][0] = 0x40000000; 570 msg->cnoc_cmds_data[1][0] = 0x60000001; 571 } 572 573 static void adreno_7c3_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 574 { 575 /* 576 * Send a single "off" entry just to get things running 577 * TODO: bus scaling 578 */ 579 msg->bw_level_num = 1; 580 581 msg->ddr_cmds_num = 3; 582 msg->ddr_wait_bitmask = 0x07; 583 584 msg->ddr_cmds_addrs[0] = 0x50004; 585 msg->ddr_cmds_addrs[1] = 0x50000; 586 msg->ddr_cmds_addrs[2] = 0x50088; 587 588 msg->ddr_cmds_data[0][0] = 0x40000000; 589 msg->ddr_cmds_data[0][1] = 0x40000000; 590 msg->ddr_cmds_data[0][2] = 0x40000000; 591 592 /* 593 * These are the CX (CNOC) votes - these are used by the GMU but the 594 * votes are known and fixed for the target 595 */ 596 msg->cnoc_cmds_num = 1; 597 msg->cnoc_wait_bitmask = 0x01; 598 599 msg->cnoc_cmds_addrs[0] = 0x5006c; 600 msg->cnoc_cmds_data[0][0] = 0x40000000; 601 msg->cnoc_cmds_data[1][0] = 0x60000001; 602 } 603 604 static void a730_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 605 { 606 msg->bw_level_num = 12; 607 608 msg->ddr_cmds_num = 3; 609 msg->ddr_wait_bitmask = 0x7; 610 611 msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0"); 612 msg->ddr_cmds_addrs[1] = cmd_db_read_addr("MC0"); 613 msg->ddr_cmds_addrs[2] = cmd_db_read_addr("ACV"); 614 615 msg->ddr_cmds_data[0][0] = 0x40000000; 616 msg->ddr_cmds_data[0][1] = 0x40000000; 617 msg->ddr_cmds_data[0][2] = 0x40000000; 618 msg->ddr_cmds_data[1][0] = 0x600002e8; 619 msg->ddr_cmds_data[1][1] = 0x600003d0; 620 msg->ddr_cmds_data[1][2] = 0x60000008; 621 msg->ddr_cmds_data[2][0] = 0x6000068d; 622 msg->ddr_cmds_data[2][1] = 0x6000089a; 623 msg->ddr_cmds_data[2][2] = 0x60000008; 624 msg->ddr_cmds_data[3][0] = 0x600007f2; 625 msg->ddr_cmds_data[3][1] = 0x60000a6e; 626 msg->ddr_cmds_data[3][2] = 0x60000008; 627 msg->ddr_cmds_data[4][0] = 0x600009e5; 628 msg->ddr_cmds_data[4][1] = 0x60000cfd; 629 msg->ddr_cmds_data[4][2] = 0x60000008; 630 msg->ddr_cmds_data[5][0] = 0x60000b29; 631 msg->ddr_cmds_data[5][1] = 0x60000ea6; 632 msg->ddr_cmds_data[5][2] = 0x60000008; 633 msg->ddr_cmds_data[6][0] = 0x60001698; 634 msg->ddr_cmds_data[6][1] = 0x60001da8; 635 msg->ddr_cmds_data[6][2] = 0x60000008; 636 msg->ddr_cmds_data[7][0] = 0x600018d2; 637 msg->ddr_cmds_data[7][1] = 0x60002093; 638 msg->ddr_cmds_data[7][2] = 0x60000008; 639 msg->ddr_cmds_data[8][0] = 0x60001e66; 640 msg->ddr_cmds_data[8][1] = 0x600027e6; 641 msg->ddr_cmds_data[8][2] = 0x60000008; 642 msg->ddr_cmds_data[9][0] = 0x600027c2; 643 msg->ddr_cmds_data[9][1] = 0x6000342f; 644 msg->ddr_cmds_data[9][2] = 0x60000008; 645 msg->ddr_cmds_data[10][0] = 0x60002e71; 646 msg->ddr_cmds_data[10][1] = 0x60003cf5; 647 msg->ddr_cmds_data[10][2] = 0x60000008; 648 msg->ddr_cmds_data[11][0] = 0x600030ae; 649 msg->ddr_cmds_data[11][1] = 0x60003fe5; 650 msg->ddr_cmds_data[11][2] = 0x60000008; 651 652 msg->cnoc_cmds_num = 1; 653 msg->cnoc_wait_bitmask = 0x1; 654 655 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0"); 656 msg->cnoc_cmds_data[0][0] = 0x40000000; 657 msg->cnoc_cmds_data[1][0] = 0x60000001; 658 } 659 660 static void a740_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 661 { 662 msg->bw_level_num = 1; 663 664 msg->ddr_cmds_num = 3; 665 msg->ddr_wait_bitmask = 0x7; 666 667 msg->ddr_cmds_addrs[0] = cmd_db_read_addr("SH0"); 668 msg->ddr_cmds_addrs[1] = cmd_db_read_addr("MC0"); 669 msg->ddr_cmds_addrs[2] = cmd_db_read_addr("ACV"); 670 671 msg->ddr_cmds_data[0][0] = 0x40000000; 672 msg->ddr_cmds_data[0][1] = 0x40000000; 673 msg->ddr_cmds_data[0][2] = 0x40000000; 674 675 /* TODO: add a proper dvfs table */ 676 677 msg->cnoc_cmds_num = 1; 678 msg->cnoc_wait_bitmask = 0x1; 679 680 msg->cnoc_cmds_addrs[0] = cmd_db_read_addr("CN0"); 681 msg->cnoc_cmds_data[0][0] = 0x40000000; 682 msg->cnoc_cmds_data[1][0] = 0x60000001; 683 } 684 685 static void a6xx_build_bw_table(struct a6xx_hfi_msg_bw_table *msg) 686 { 687 /* Send a single "off" entry since the 630 GMU doesn't do bus scaling */ 688 msg->bw_level_num = 1; 689 690 msg->ddr_cmds_num = 3; 691 msg->ddr_wait_bitmask = 0x07; 692 693 msg->ddr_cmds_addrs[0] = 0x50000; 694 msg->ddr_cmds_addrs[1] = 0x5005c; 695 msg->ddr_cmds_addrs[2] = 0x5000c; 696 697 msg->ddr_cmds_data[0][0] = 0x40000000; 698 msg->ddr_cmds_data[0][1] = 0x40000000; 699 msg->ddr_cmds_data[0][2] = 0x40000000; 700 701 /* 702 * These are the CX (CNOC) votes. This is used but the values for the 703 * sdm845 GMU are known and fixed so we can hard code them. 704 */ 705 706 msg->cnoc_cmds_num = 3; 707 msg->cnoc_wait_bitmask = 0x05; 708 709 msg->cnoc_cmds_addrs[0] = 0x50034; 710 msg->cnoc_cmds_addrs[1] = 0x5007c; 711 msg->cnoc_cmds_addrs[2] = 0x5004c; 712 713 msg->cnoc_cmds_data[0][0] = 0x40000000; 714 msg->cnoc_cmds_data[0][1] = 0x00000000; 715 msg->cnoc_cmds_data[0][2] = 0x40000000; 716 717 msg->cnoc_cmds_data[1][0] = 0x60000001; 718 msg->cnoc_cmds_data[1][1] = 0x20000001; 719 msg->cnoc_cmds_data[1][2] = 0x60000001; 720 } 721 722 723 static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu) 724 { 725 struct a6xx_hfi_msg_bw_table *msg; 726 struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); 727 struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; 728 const struct a6xx_info *info = adreno_gpu->info->a6xx; 729 730 if (gmu->bw_table) 731 goto send; 732 733 msg = devm_kzalloc(gmu->dev, sizeof(*msg), GFP_KERNEL); 734 if (!msg) 735 return -ENOMEM; 736 737 if (info->bcms && gmu->nr_gpu_bws > 1) 738 a6xx_generate_bw_table(info, gmu, msg); 739 else if (adreno_is_a618(adreno_gpu)) 740 a618_build_bw_table(msg); 741 else if (adreno_is_a619(adreno_gpu)) 742 a619_build_bw_table(msg); 743 else if (adreno_is_a640_family(adreno_gpu)) 744 a640_build_bw_table(msg); 745 else if (adreno_is_a650(adreno_gpu)) 746 a650_build_bw_table(msg); 747 else if (adreno_is_7c3(adreno_gpu)) 748 adreno_7c3_build_bw_table(msg); 749 else if (adreno_is_a660(adreno_gpu)) 750 a660_build_bw_table(msg); 751 else if (adreno_is_a663(adreno_gpu)) 752 a663_build_bw_table(msg); 753 else if (adreno_is_a690(adreno_gpu)) 754 a690_build_bw_table(msg); 755 else if (adreno_is_a730(adreno_gpu)) 756 a730_build_bw_table(msg); 757 else if (adreno_is_a740_family(adreno_gpu)) 758 a740_build_bw_table(msg); 759 else 760 a6xx_build_bw_table(msg); 761 762 gmu->bw_table = msg; 763 764 send: 765 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, gmu->bw_table, sizeof(*(gmu->bw_table)), 766 NULL, 0); 767 } 768 769 static int a6xx_hfi_feature_ctrl_msg(struct a6xx_gmu *gmu, u32 feature, u32 enable, u32 data) 770 { 771 struct a6xx_hfi_msg_feature_ctrl msg = { 772 .feature = feature, 773 .enable = enable, 774 .data = data, 775 }; 776 777 return a6xx_hfi_send_msg(gmu, HFI_H2F_FEATURE_CTRL, &msg, sizeof(msg), NULL, 0); 778 } 779 780 #define HFI_FEATURE_IFPC 9 781 #define IFPC_LONG_HYST 0x1680 782 783 static int a6xx_hfi_enable_ifpc(struct a6xx_gmu *gmu) 784 { 785 if (gmu->idle_level != GMU_IDLE_STATE_IFPC) 786 return 0; 787 788 return a6xx_hfi_feature_ctrl_msg(gmu, HFI_FEATURE_IFPC, 1, IFPC_LONG_HYST); 789 } 790 791 #define HFI_FEATURE_ACD 12 792 793 static int a6xx_hfi_enable_acd(struct a6xx_gmu *gmu) 794 { 795 struct a6xx_hfi_acd_table *acd_table = &gmu->acd_table; 796 int ret; 797 798 if (!acd_table->enable_by_level) 799 return 0; 800 801 /* Enable ACD feature at GMU */ 802 ret = a6xx_hfi_feature_ctrl_msg(gmu, HFI_FEATURE_ACD, 1, 0); 803 if (ret) { 804 DRM_DEV_ERROR(gmu->dev, "Unable to enable ACD (%d)\n", ret); 805 return ret; 806 } 807 808 /* Send ACD table to GMU */ 809 ret = a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_ACD, acd_table, sizeof(*acd_table), NULL, 0); 810 if (ret) { 811 DRM_DEV_ERROR(gmu->dev, "Unable to ACD table (%d)\n", ret); 812 return ret; 813 } 814 815 return 0; 816 } 817 818 static int a6xx_hfi_send_test(struct a6xx_gmu *gmu) 819 { 820 struct a6xx_hfi_msg_test msg = { 0 }; 821 822 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg), 823 NULL, 0); 824 } 825 826 static int a6xx_hfi_send_start(struct a6xx_gmu *gmu) 827 { 828 struct a6xx_hfi_msg_start msg = { 0 }; 829 830 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_START, &msg, sizeof(msg), 831 NULL, 0); 832 } 833 834 static int a6xx_hfi_send_core_fw_start(struct a6xx_gmu *gmu) 835 { 836 struct a6xx_hfi_msg_core_fw_start msg = { 0 }; 837 838 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_CORE_FW_START, &msg, 839 sizeof(msg), NULL, 0); 840 } 841 842 int a6xx_hfi_set_freq(struct a6xx_gmu *gmu, u32 freq_index, u32 bw_index) 843 { 844 struct a6xx_hfi_gx_bw_perf_vote_cmd msg = { 0 }; 845 846 msg.ack_type = 1; /* blocking */ 847 msg.freq = freq_index; 848 msg.bw = bw_index; 849 850 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_GX_BW_PERF_VOTE, &msg, 851 sizeof(msg), NULL, 0); 852 } 853 854 int a6xx_hfi_send_prep_slumber(struct a6xx_gmu *gmu) 855 { 856 struct a6xx_hfi_prep_slumber_cmd msg = { 0 }; 857 858 /* TODO: should freq and bw fields be non-zero ? */ 859 860 return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PREPARE_SLUMBER, &msg, 861 sizeof(msg), NULL, 0); 862 } 863 864 static int a6xx_hfi_start_v1(struct a6xx_gmu *gmu, int boot_state) 865 { 866 int ret; 867 868 ret = a6xx_hfi_send_gmu_init(gmu, boot_state); 869 if (ret) 870 return ret; 871 872 ret = a6xx_hfi_get_fw_version(gmu, NULL); 873 if (ret) 874 return ret; 875 876 /* 877 * We have to get exchange version numbers per the sequence but at this 878 * point th kernel driver doesn't need to know the exact version of 879 * the GMU firmware 880 */ 881 882 ret = a6xx_hfi_send_perf_table_v1(gmu); 883 if (ret) 884 return ret; 885 886 ret = a6xx_hfi_send_bw_table(gmu); 887 if (ret) 888 return ret; 889 890 /* 891 * Let the GMU know that there won't be any more HFI messages until next 892 * boot 893 */ 894 a6xx_hfi_send_test(gmu); 895 896 return 0; 897 } 898 899 int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) 900 { 901 int ret; 902 903 if (gmu->legacy) 904 return a6xx_hfi_start_v1(gmu, boot_state); 905 906 907 ret = a6xx_hfi_send_perf_table(gmu); 908 if (ret) 909 return ret; 910 911 ret = a6xx_hfi_send_bw_table(gmu); 912 if (ret) 913 return ret; 914 915 ret = a6xx_hfi_enable_acd(gmu); 916 if (ret) 917 return ret; 918 919 ret = a6xx_hfi_enable_ifpc(gmu); 920 if (ret) 921 return ret; 922 923 ret = a6xx_hfi_send_core_fw_start(gmu); 924 if (ret) 925 return ret; 926 927 /* 928 * Downstream driver sends this in its "a6xx_hw_init" equivalent, 929 * but seems to be no harm in sending it here 930 */ 931 ret = a6xx_hfi_send_start(gmu); 932 if (ret) 933 return ret; 934 935 return 0; 936 } 937 938 void a6xx_hfi_stop(struct a6xx_gmu *gmu) 939 { 940 int i; 941 942 for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) { 943 struct a6xx_hfi_queue *queue = &gmu->queues[i]; 944 945 if (!queue->header) 946 continue; 947 948 if (queue->header->read_index != queue->header->write_index) 949 DRM_DEV_ERROR(gmu->dev, "HFI queue %d is not empty\n", i); 950 951 queue->header->read_index = 0; 952 queue->header->write_index = 0; 953 954 memset(&queue->history, 0xff, sizeof(queue->history)); 955 queue->history_idx = 0; 956 } 957 } 958 959 static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue, 960 struct a6xx_hfi_queue_header *header, void *virt, u64 iova, 961 u32 id) 962 { 963 spin_lock_init(&queue->lock); 964 queue->header = header; 965 queue->data = virt; 966 atomic_set(&queue->seqnum, 0); 967 968 memset(&queue->history, 0xff, sizeof(queue->history)); 969 queue->history_idx = 0; 970 971 /* Set up the shared memory header */ 972 header->iova = iova; 973 header->type = 10 << 8 | id; 974 header->status = 1; 975 header->size = SZ_4K >> 2; 976 header->msg_size = 0; 977 header->dropped = 0; 978 header->rx_watermark = 1; 979 header->tx_watermark = 1; 980 header->rx_request = 1; 981 header->tx_request = 0; 982 header->read_index = 0; 983 header->write_index = 0; 984 } 985 986 void a6xx_hfi_init(struct a6xx_gmu *gmu) 987 { 988 struct a6xx_gmu_bo *hfi = &gmu->hfi; 989 struct a6xx_hfi_queue_table_header *table = hfi->virt; 990 struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table); 991 u64 offset; 992 int table_size; 993 994 /* 995 * The table size is the size of the table header plus all of the queue 996 * headers 997 */ 998 table_size = sizeof(*table); 999 table_size += (ARRAY_SIZE(gmu->queues) * 1000 sizeof(struct a6xx_hfi_queue_header)); 1001 1002 table->version = 0; 1003 table->size = table_size; 1004 /* First queue header is located immediately after the table header */ 1005 table->qhdr0_offset = sizeof(*table) >> 2; 1006 table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2; 1007 table->num_queues = ARRAY_SIZE(gmu->queues); 1008 table->active_queues = ARRAY_SIZE(gmu->queues); 1009 1010 /* Command queue */ 1011 offset = SZ_4K; 1012 a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset, 1013 hfi->iova + offset, 0); 1014 1015 /* GMU response queue */ 1016 offset += SZ_4K; 1017 a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset, 1018 hfi->iova + offset, gmu->legacy ? 4 : 1); 1019 } 1020