1 // SPDX-License-Identifier: GPL-2.0 2 // 3 // Copyright (c) 2018 MediaTek Inc. 4 5 #include <linux/completion.h> 6 #include <linux/errno.h> 7 #include <linux/dma-mapping.h> 8 #include <linux/module.h> 9 #include <linux/mailbox_controller.h> 10 #include <linux/of.h> 11 #include <linux/of_address.h> 12 #include <linux/soc/mediatek/mtk-cmdq.h> 13 14 #define CMDQ_WRITE_ENABLE_MASK BIT(0) 15 #define CMDQ_POLL_ENABLE_MASK BIT(0) 16 /* dedicate the last GPR_R15 to assign the register address to be poll */ 17 #define CMDQ_POLL_ADDR_GPR (15) 18 #define CMDQ_EOC_IRQ_EN BIT(0) 19 #define CMDQ_IMMEDIATE_VALUE 0 20 #define CMDQ_REG_TYPE 1 21 #define CMDQ_JUMP_RELATIVE 0 22 #define CMDQ_JUMP_ABSOLUTE 1 23 24 struct cmdq_instruction { 25 union { 26 u32 value; 27 u32 mask; 28 struct { 29 u16 arg_c; 30 u16 src_reg; 31 }; 32 }; 33 union { 34 u16 offset; 35 u16 event; 36 u16 reg_dst; 37 }; 38 union { 39 u8 subsys; 40 struct { 41 u8 sop:5; 42 u8 arg_c_t:1; 43 u8 src_t:1; 44 u8 dst_t:1; 45 }; 46 }; 47 u8 op; 48 }; 49 50 static inline u8 cmdq_operand_get_type(struct cmdq_operand *op) 51 { 52 return op->reg ? CMDQ_REG_TYPE : CMDQ_IMMEDIATE_VALUE; 53 } 54 55 static inline u16 cmdq_operand_get_idx_value(struct cmdq_operand *op) 56 { 57 return op->reg ? op->idx : op->value; 58 } 59 60 int cmdq_dev_get_client_reg(struct device *dev, 61 struct cmdq_client_reg *client_reg, int idx) 62 { 63 struct of_phandle_args spec; 64 struct resource res; 65 int err; 66 67 if (!client_reg) 68 return -ENOENT; 69 70 err = of_address_to_resource(dev->of_node, 0, &res); 71 if (err) { 72 dev_err(dev, "Missing reg in %s node\n", dev->of_node->full_name); 73 return -EINVAL; 74 } 75 client_reg->pa_base = res.start; 76 77 err = of_parse_phandle_with_fixed_args(dev->of_node, 78 "mediatek,gce-client-reg", 79 3, idx, &spec); 80 if (err < 0) { 81 dev_dbg(dev, 82 "error %d can't parse gce-client-reg property (%d)", 83 err, idx); 84 85 /* make subsys invalid */ 86 client_reg->subsys = CMDQ_SUBSYS_INVALID; 87 88 /* 89 * All GCEs support writing register PA with mask without subsys, 90 * but this requires extra GCE instructions to convert the PA into 91 * a format that GCE can handle, which is less performance than 92 * directly using subsys. Therefore, when subsys is available, 93 * we prefer to use subsys for writing register PA. 94 */ 95 client_reg->pkt_write = cmdq_pkt_write_pa; 96 client_reg->pkt_write_mask = cmdq_pkt_write_mask_pa; 97 98 return 0; 99 } 100 101 client_reg->subsys = (u8)spec.args[0]; 102 client_reg->offset = (u16)spec.args[1]; 103 client_reg->size = (u16)spec.args[2]; 104 of_node_put(spec.np); 105 106 client_reg->pkt_write = cmdq_pkt_write_subsys; 107 client_reg->pkt_write_mask = cmdq_pkt_write_mask_subsys; 108 109 return 0; 110 } 111 EXPORT_SYMBOL(cmdq_dev_get_client_reg); 112 113 struct cmdq_client *cmdq_mbox_create(struct device *dev, int index) 114 { 115 struct cmdq_client *client; 116 117 client = kzalloc_obj(*client); 118 if (!client) 119 return (struct cmdq_client *)-ENOMEM; 120 121 client->client.dev = dev; 122 client->client.tx_block = false; 123 client->client.knows_txdone = true; 124 client->chan = mbox_request_channel(&client->client, index); 125 126 if (IS_ERR(client->chan)) { 127 long err; 128 129 dev_err(dev, "failed to request channel\n"); 130 err = PTR_ERR(client->chan); 131 kfree(client); 132 133 return ERR_PTR(err); 134 } 135 136 return client; 137 } 138 EXPORT_SYMBOL(cmdq_mbox_create); 139 140 void cmdq_mbox_destroy(struct cmdq_client *client) 141 { 142 mbox_free_channel(client->chan); 143 kfree(client); 144 } 145 EXPORT_SYMBOL(cmdq_mbox_destroy); 146 147 int cmdq_pkt_create(struct cmdq_client *client, struct cmdq_pkt *pkt, size_t size) 148 { 149 struct device *dev; 150 dma_addr_t dma_addr; 151 152 pkt->va_base = kzalloc(size, GFP_KERNEL); 153 if (!pkt->va_base) 154 return -ENOMEM; 155 156 pkt->buf_size = size; 157 158 dev = client->chan->mbox->dev; 159 dma_addr = dma_map_single(dev, pkt->va_base, pkt->buf_size, 160 DMA_TO_DEVICE); 161 if (dma_mapping_error(dev, dma_addr)) { 162 dev_err(dev, "dma map failed, size=%u\n", (u32)(u64)size); 163 kfree(pkt->va_base); 164 return -ENOMEM; 165 } 166 167 pkt->pa_base = dma_addr; 168 cmdq_get_mbox_priv(client->chan, &pkt->priv); 169 170 return 0; 171 } 172 EXPORT_SYMBOL(cmdq_pkt_create); 173 174 void cmdq_pkt_destroy(struct cmdq_client *client, struct cmdq_pkt *pkt) 175 { 176 dma_unmap_single(client->chan->mbox->dev, pkt->pa_base, pkt->buf_size, 177 DMA_TO_DEVICE); 178 kfree(pkt->va_base); 179 } 180 EXPORT_SYMBOL(cmdq_pkt_destroy); 181 182 static int cmdq_pkt_append_command(struct cmdq_pkt *pkt, 183 struct cmdq_instruction inst) 184 { 185 struct cmdq_instruction *cmd_ptr; 186 187 if (unlikely(pkt->cmd_buf_size + CMDQ_INST_SIZE > pkt->buf_size)) { 188 /* 189 * In the case of allocated buffer size (pkt->buf_size) is used 190 * up, the real required size (pkt->cmdq_buf_size) is still 191 * increased, so that the user knows how much memory should be 192 * ultimately allocated after appending all commands and 193 * flushing the command packet. Therefor, the user can call 194 * cmdq_pkt_create() again with the real required buffer size. 195 */ 196 pkt->cmd_buf_size += CMDQ_INST_SIZE; 197 WARN_ONCE(1, "%s: buffer size %u is too small !\n", 198 __func__, (u32)pkt->buf_size); 199 return -ENOMEM; 200 } 201 202 cmd_ptr = pkt->va_base + pkt->cmd_buf_size; 203 *cmd_ptr = inst; 204 pkt->cmd_buf_size += CMDQ_INST_SIZE; 205 206 return 0; 207 } 208 209 static int cmdq_pkt_mask(struct cmdq_pkt *pkt, u32 mask) 210 { 211 struct cmdq_instruction inst = { 212 .op = CMDQ_CODE_MASK, 213 .mask = ~mask 214 }; 215 return cmdq_pkt_append_command(pkt, inst); 216 } 217 218 int cmdq_pkt_write(struct cmdq_pkt *pkt, u8 subsys, u16 offset, u32 value) 219 { 220 struct cmdq_instruction inst = { 221 .op = CMDQ_CODE_WRITE, 222 .value = value, 223 .offset = offset, 224 .subsys = subsys 225 }; 226 return cmdq_pkt_append_command(pkt, inst); 227 } 228 EXPORT_SYMBOL(cmdq_pkt_write); 229 230 int cmdq_pkt_write_pa(struct cmdq_pkt *pkt, u8 subsys /*unused*/, u32 pa_base, 231 u16 offset, u32 value) 232 { 233 int err; 234 235 err = cmdq_pkt_assign(pkt, CMDQ_THR_SPR_IDX0, CMDQ_ADDR_HIGH(pa_base)); 236 if (err < 0) 237 return err; 238 239 return cmdq_pkt_write_s_value(pkt, CMDQ_THR_SPR_IDX0, CMDQ_ADDR_LOW(offset), value); 240 } 241 EXPORT_SYMBOL(cmdq_pkt_write_pa); 242 243 int cmdq_pkt_write_subsys(struct cmdq_pkt *pkt, u8 subsys, u32 pa_base /*unused*/, 244 u16 offset, u32 value) 245 { 246 return cmdq_pkt_write(pkt, subsys, offset, value); 247 } 248 EXPORT_SYMBOL(cmdq_pkt_write_subsys); 249 250 int cmdq_pkt_write_mask(struct cmdq_pkt *pkt, u8 subsys, 251 u16 offset, u32 value, u32 mask) 252 { 253 u16 offset_mask = offset; 254 int err; 255 256 if (mask != GENMASK(31, 0)) { 257 err = cmdq_pkt_mask(pkt, mask); 258 if (err < 0) 259 return err; 260 261 offset_mask |= CMDQ_WRITE_ENABLE_MASK; 262 } 263 return cmdq_pkt_write(pkt, subsys, offset_mask, value); 264 } 265 EXPORT_SYMBOL(cmdq_pkt_write_mask); 266 267 int cmdq_pkt_write_mask_pa(struct cmdq_pkt *pkt, u8 subsys /*unused*/, u32 pa_base, 268 u16 offset, u32 value, u32 mask) 269 { 270 int err; 271 272 err = cmdq_pkt_assign(pkt, CMDQ_THR_SPR_IDX0, CMDQ_ADDR_HIGH(pa_base)); 273 if (err < 0) 274 return err; 275 276 return cmdq_pkt_write_s_mask_value(pkt, CMDQ_THR_SPR_IDX0, 277 CMDQ_ADDR_LOW(offset), value, mask); 278 } 279 EXPORT_SYMBOL(cmdq_pkt_write_mask_pa); 280 281 int cmdq_pkt_write_mask_subsys(struct cmdq_pkt *pkt, u8 subsys, u32 pa_base /*unused*/, 282 u16 offset, u32 value, u32 mask) 283 { 284 return cmdq_pkt_write_mask(pkt, subsys, offset, value, mask); 285 } 286 EXPORT_SYMBOL(cmdq_pkt_write_mask_subsys); 287 288 int cmdq_pkt_read_s(struct cmdq_pkt *pkt, u16 high_addr_reg_idx, u16 addr_low, 289 u16 reg_idx) 290 { 291 struct cmdq_instruction inst = { 292 .op = CMDQ_CODE_READ_S, 293 .dst_t = CMDQ_REG_TYPE, 294 .sop = high_addr_reg_idx, 295 .reg_dst = reg_idx, 296 .src_reg = addr_low 297 }; 298 return cmdq_pkt_append_command(pkt, inst); 299 } 300 EXPORT_SYMBOL(cmdq_pkt_read_s); 301 302 int cmdq_pkt_write_s(struct cmdq_pkt *pkt, u16 high_addr_reg_idx, 303 u16 addr_low, u16 src_reg_idx) 304 { 305 struct cmdq_instruction inst = { 306 .op = CMDQ_CODE_WRITE_S, 307 .src_t = CMDQ_REG_TYPE, 308 .sop = high_addr_reg_idx, 309 .offset = addr_low, 310 .src_reg = src_reg_idx 311 }; 312 return cmdq_pkt_append_command(pkt, inst); 313 } 314 EXPORT_SYMBOL(cmdq_pkt_write_s); 315 316 int cmdq_pkt_write_s_mask(struct cmdq_pkt *pkt, u16 high_addr_reg_idx, 317 u16 addr_low, u16 src_reg_idx, u32 mask) 318 { 319 struct cmdq_instruction inst = { 320 .op = CMDQ_CODE_WRITE_S_MASK, 321 .src_t = CMDQ_REG_TYPE, 322 .sop = high_addr_reg_idx, 323 .offset = addr_low, 324 .src_reg = src_reg_idx, 325 }; 326 int err; 327 328 err = cmdq_pkt_mask(pkt, mask); 329 if (err < 0) 330 return err; 331 332 return cmdq_pkt_append_command(pkt, inst); 333 } 334 EXPORT_SYMBOL(cmdq_pkt_write_s_mask); 335 336 int cmdq_pkt_write_s_value(struct cmdq_pkt *pkt, u8 high_addr_reg_idx, 337 u16 addr_low, u32 value) 338 { 339 struct cmdq_instruction inst = { 340 .op = CMDQ_CODE_WRITE_S, 341 .sop = high_addr_reg_idx, 342 .offset = addr_low, 343 .value = value 344 }; 345 return cmdq_pkt_append_command(pkt, inst); 346 } 347 EXPORT_SYMBOL(cmdq_pkt_write_s_value); 348 349 int cmdq_pkt_write_s_mask_value(struct cmdq_pkt *pkt, u8 high_addr_reg_idx, 350 u16 addr_low, u32 value, u32 mask) 351 { 352 struct cmdq_instruction inst = { 353 .op = CMDQ_CODE_WRITE_S_MASK, 354 .sop = high_addr_reg_idx, 355 .offset = addr_low, 356 .value = value 357 }; 358 int err; 359 360 err = cmdq_pkt_mask(pkt, mask); 361 if (err < 0) 362 return err; 363 364 return cmdq_pkt_append_command(pkt, inst); 365 } 366 EXPORT_SYMBOL(cmdq_pkt_write_s_mask_value); 367 368 int cmdq_pkt_mem_move(struct cmdq_pkt *pkt, dma_addr_t src_addr, dma_addr_t dst_addr) 369 { 370 const u16 high_addr_reg_idx = CMDQ_THR_SPR_IDX0; 371 const u16 value_reg_idx = CMDQ_THR_SPR_IDX1; 372 int ret; 373 374 /* read the value of src_addr into high_addr_reg_idx */ 375 src_addr += pkt->priv.mminfra_offset; 376 ret = cmdq_pkt_assign(pkt, high_addr_reg_idx, CMDQ_ADDR_HIGH(src_addr)); 377 if (ret < 0) 378 return ret; 379 ret = cmdq_pkt_read_s(pkt, high_addr_reg_idx, CMDQ_ADDR_LOW(src_addr), value_reg_idx); 380 if (ret < 0) 381 return ret; 382 383 /* write the value of value_reg_idx into dst_addr */ 384 dst_addr += pkt->priv.mminfra_offset; 385 ret = cmdq_pkt_assign(pkt, high_addr_reg_idx, CMDQ_ADDR_HIGH(dst_addr)); 386 if (ret < 0) 387 return ret; 388 ret = cmdq_pkt_write_s(pkt, high_addr_reg_idx, CMDQ_ADDR_LOW(dst_addr), value_reg_idx); 389 if (ret < 0) 390 return ret; 391 392 return 0; 393 } 394 EXPORT_SYMBOL(cmdq_pkt_mem_move); 395 396 int cmdq_pkt_wfe(struct cmdq_pkt *pkt, u16 event, bool clear) 397 { 398 u32 clear_option = clear ? CMDQ_WFE_UPDATE : 0; 399 struct cmdq_instruction inst = { 400 .op = CMDQ_CODE_WFE, 401 .value = CMDQ_WFE_OPTION | clear_option, 402 .event = event 403 }; 404 405 if (event >= CMDQ_MAX_EVENT) 406 return -EINVAL; 407 408 return cmdq_pkt_append_command(pkt, inst); 409 } 410 EXPORT_SYMBOL(cmdq_pkt_wfe); 411 412 int cmdq_pkt_acquire_event(struct cmdq_pkt *pkt, u16 event) 413 { 414 struct cmdq_instruction inst = { 415 .op = CMDQ_CODE_WFE, 416 .value = CMDQ_WFE_UPDATE | CMDQ_WFE_UPDATE_VALUE | CMDQ_WFE_WAIT, 417 .event = event 418 }; 419 420 if (event >= CMDQ_MAX_EVENT) 421 return -EINVAL; 422 423 return cmdq_pkt_append_command(pkt, inst); 424 } 425 EXPORT_SYMBOL(cmdq_pkt_acquire_event); 426 427 int cmdq_pkt_clear_event(struct cmdq_pkt *pkt, u16 event) 428 { 429 struct cmdq_instruction inst = { 430 .op = CMDQ_CODE_WFE, 431 .value = CMDQ_WFE_UPDATE, 432 .event = event 433 }; 434 435 if (event >= CMDQ_MAX_EVENT) 436 return -EINVAL; 437 438 return cmdq_pkt_append_command(pkt, inst); 439 } 440 EXPORT_SYMBOL(cmdq_pkt_clear_event); 441 442 int cmdq_pkt_set_event(struct cmdq_pkt *pkt, u16 event) 443 { 444 struct cmdq_instruction inst = { 445 .op = CMDQ_CODE_WFE, 446 .value = CMDQ_WFE_UPDATE | CMDQ_WFE_UPDATE_VALUE, 447 .event = event 448 }; 449 450 if (event >= CMDQ_MAX_EVENT) 451 return -EINVAL; 452 453 return cmdq_pkt_append_command(pkt, inst); 454 } 455 EXPORT_SYMBOL(cmdq_pkt_set_event); 456 457 int cmdq_pkt_poll(struct cmdq_pkt *pkt, u8 subsys, 458 u16 offset, u32 value) 459 { 460 struct cmdq_instruction inst = { 461 .op = CMDQ_CODE_POLL, 462 .value = value, 463 .offset = offset, 464 .subsys = subsys 465 }; 466 return cmdq_pkt_append_command(pkt, inst); 467 } 468 EXPORT_SYMBOL(cmdq_pkt_poll); 469 470 int cmdq_pkt_poll_mask(struct cmdq_pkt *pkt, u8 subsys, 471 u16 offset, u32 value, u32 mask) 472 { 473 int err; 474 475 err = cmdq_pkt_mask(pkt, mask); 476 if (err < 0) 477 return err; 478 479 offset = offset | CMDQ_POLL_ENABLE_MASK; 480 return cmdq_pkt_poll(pkt, subsys, offset, value); 481 } 482 EXPORT_SYMBOL(cmdq_pkt_poll_mask); 483 484 int cmdq_pkt_poll_addr(struct cmdq_pkt *pkt, dma_addr_t addr, u32 value, u32 mask) 485 { 486 struct cmdq_instruction inst = { {0} }; 487 u8 use_mask = 0; 488 int ret; 489 490 /* 491 * Append an MASK instruction to set the mask for following POLL instruction 492 * which enables use_mask bit. 493 */ 494 if (mask != GENMASK(31, 0)) { 495 ret = cmdq_pkt_mask(pkt, mask); 496 if (ret < 0) 497 return ret; 498 use_mask = CMDQ_POLL_ENABLE_MASK; 499 } 500 501 /* 502 * POLL is an legacy operation in GCE and it does not support SPR and CMDQ_CODE_LOGIC, 503 * so it can not use cmdq_pkt_assign to keep polling register address to SPR. 504 * If user wants to poll a register address which doesn't have a subsys id, 505 * user needs to use GPR and CMDQ_CODE_MASK to move polling register address to GPR. 506 */ 507 inst.op = CMDQ_CODE_MASK; 508 inst.dst_t = CMDQ_REG_TYPE; 509 inst.sop = CMDQ_POLL_ADDR_GPR; 510 inst.value = addr + pkt->priv.mminfra_offset; 511 ret = cmdq_pkt_append_command(pkt, inst); 512 if (ret < 0) 513 return ret; 514 515 /* Append POLL instruction to poll the register address assign to GPR previously. */ 516 inst.op = CMDQ_CODE_POLL; 517 inst.dst_t = CMDQ_REG_TYPE; 518 inst.sop = CMDQ_POLL_ADDR_GPR; 519 inst.offset = use_mask; 520 inst.value = value; 521 ret = cmdq_pkt_append_command(pkt, inst); 522 if (ret < 0) 523 return ret; 524 525 return 0; 526 } 527 EXPORT_SYMBOL(cmdq_pkt_poll_addr); 528 529 int cmdq_pkt_logic_command(struct cmdq_pkt *pkt, u16 result_reg_idx, 530 struct cmdq_operand *left_operand, 531 enum cmdq_logic_op s_op, 532 struct cmdq_operand *right_operand) 533 { 534 struct cmdq_instruction inst; 535 536 if (!left_operand || !right_operand || s_op >= CMDQ_LOGIC_MAX) 537 return -EINVAL; 538 539 inst.value = 0; 540 inst.op = CMDQ_CODE_LOGIC; 541 inst.dst_t = CMDQ_REG_TYPE; 542 inst.src_t = cmdq_operand_get_type(left_operand); 543 inst.arg_c_t = cmdq_operand_get_type(right_operand); 544 inst.sop = s_op; 545 inst.reg_dst = result_reg_idx; 546 inst.src_reg = cmdq_operand_get_idx_value(left_operand); 547 inst.arg_c = cmdq_operand_get_idx_value(right_operand); 548 549 return cmdq_pkt_append_command(pkt, inst); 550 } 551 EXPORT_SYMBOL(cmdq_pkt_logic_command); 552 553 int cmdq_pkt_assign(struct cmdq_pkt *pkt, u16 reg_idx, u32 value) 554 { 555 struct cmdq_instruction inst = { 556 .op = CMDQ_CODE_LOGIC, 557 .dst_t = CMDQ_REG_TYPE, 558 .reg_dst = reg_idx, 559 .value = value 560 }; 561 return cmdq_pkt_append_command(pkt, inst); 562 } 563 EXPORT_SYMBOL(cmdq_pkt_assign); 564 565 int cmdq_pkt_jump_abs(struct cmdq_pkt *pkt, dma_addr_t addr, u8 shift_pa) 566 { 567 struct cmdq_instruction inst = { 568 .op = CMDQ_CODE_JUMP, 569 .offset = CMDQ_JUMP_ABSOLUTE, 570 .value = (addr + pkt->priv.mminfra_offset) >> pkt->priv.shift_pa 571 }; 572 return cmdq_pkt_append_command(pkt, inst); 573 } 574 EXPORT_SYMBOL(cmdq_pkt_jump_abs); 575 576 int cmdq_pkt_jump_rel(struct cmdq_pkt *pkt, s32 offset, u8 shift_pa) 577 { 578 struct cmdq_instruction inst = { 579 .op = CMDQ_CODE_JUMP, 580 .value = (u32)offset >> shift_pa 581 }; 582 return cmdq_pkt_append_command(pkt, inst); 583 } 584 EXPORT_SYMBOL(cmdq_pkt_jump_rel); 585 586 int cmdq_pkt_eoc(struct cmdq_pkt *pkt) 587 { 588 struct cmdq_instruction inst = { 589 .op = CMDQ_CODE_EOC, 590 .value = CMDQ_EOC_IRQ_EN 591 }; 592 return cmdq_pkt_append_command(pkt, inst); 593 } 594 EXPORT_SYMBOL(cmdq_pkt_eoc); 595 596 MODULE_DESCRIPTION("MediaTek Command Queue (CMDQ) driver"); 597 MODULE_LICENSE("GPL v2"); 598