1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2022-2024, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/drm_device.h> 7 #include <drm/drm_managed.h> 8 #include <linux/bitfield.h> 9 #include <linux/interrupt.h> 10 #include <linux/iopoll.h> 11 #include <linux/slab.h> 12 #include <linux/xarray.h> 13 14 #define CREATE_TRACE_POINTS 15 #include <trace/events/amdxdna.h> 16 17 #include "amdxdna_mailbox.h" 18 19 #define MB_ERR(chann, fmt, args...) \ 20 ({ \ 21 typeof(chann) _chann = chann; \ 22 dev_err((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \ 23 (_chann)->msix_irq, ##args); \ 24 }) 25 #define MB_DBG(chann, fmt, args...) \ 26 ({ \ 27 typeof(chann) _chann = chann; \ 28 dev_dbg((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \ 29 (_chann)->msix_irq, ##args); \ 30 }) 31 #define MB_WARN_ONCE(chann, fmt, args...) \ 32 ({ \ 33 typeof(chann) _chann = chann; \ 34 dev_warn_once((_chann)->mb->dev, "xdna_mailbox.%d: "fmt, \ 35 (_chann)->msix_irq, ##args); \ 36 }) 37 38 #define MAGIC_VAL 0x1D000000U 39 #define MAGIC_VAL_MASK 0xFF000000 40 #define MAX_MSG_ID_ENTRIES 256 41 #define MSG_RX_TIMER 200 /* milliseconds */ 42 #define MAILBOX_NAME "xdna_mailbox" 43 44 enum channel_res_type { 45 CHAN_RES_X2I, 46 CHAN_RES_I2X, 47 CHAN_RES_NUM 48 }; 49 50 struct mailbox { 51 struct device *dev; 52 struct xdna_mailbox_res res; 53 }; 54 55 struct mailbox_channel { 56 struct mailbox *mb; 57 struct xdna_mailbox_chann_res res[CHAN_RES_NUM]; 58 int msix_irq; 59 u32 iohub_int_addr; 60 struct xarray chan_xa; 61 u32 next_msgid; 62 u32 x2i_tail; 63 64 /* Received msg related fields */ 65 struct workqueue_struct *work_q; 66 struct work_struct rx_work; 67 u32 i2x_head; 68 bool bad_state; 69 }; 70 71 #define MSG_BODY_SZ GENMASK(10, 0) 72 #define MSG_PROTO_VER GENMASK(23, 16) 73 struct xdna_msg_header { 74 __u32 total_size; 75 __u32 sz_ver; 76 __u32 id; 77 __u32 opcode; 78 } __packed; 79 80 static_assert(sizeof(struct xdna_msg_header) == 16); 81 82 struct mailbox_pkg { 83 struct xdna_msg_header header; 84 __u32 payload[]; 85 }; 86 87 /* The protocol version. */ 88 #define MSG_PROTOCOL_VERSION 0x1 89 /* The tombstone value. */ 90 #define TOMBSTONE 0xDEADFACE 91 92 struct mailbox_msg { 93 void *handle; 94 int (*notify_cb)(void *handle, void __iomem *data, size_t size); 95 size_t pkg_size; /* package size in bytes */ 96 struct mailbox_pkg pkg; 97 }; 98 99 static void mailbox_reg_write(struct mailbox_channel *mb_chann, u32 mbox_reg, u32 data) 100 { 101 struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; 102 void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; 103 104 writel(data, ringbuf_addr); 105 } 106 107 static u32 mailbox_reg_read(struct mailbox_channel *mb_chann, u32 mbox_reg) 108 { 109 struct xdna_mailbox_res *mb_res = &mb_chann->mb->res; 110 void __iomem *ringbuf_addr = mb_res->mbox_base + mbox_reg; 111 112 return readl(ringbuf_addr); 113 } 114 115 static inline void mailbox_irq_acknowledge(struct mailbox_channel *mb_chann) 116 { 117 if (mb_chann->iohub_int_addr) 118 mailbox_reg_write(mb_chann, mb_chann->iohub_int_addr, 0); 119 } 120 121 static inline u32 mailbox_irq_status(struct mailbox_channel *mb_chann) 122 { 123 return (mb_chann->iohub_int_addr) ? 124 mailbox_reg_read(mb_chann, mb_chann->iohub_int_addr) : 0; 125 } 126 127 static inline void 128 mailbox_set_headptr(struct mailbox_channel *mb_chann, u32 headptr_val) 129 { 130 mailbox_reg_write(mb_chann, mb_chann->res[CHAN_RES_I2X].mb_head_ptr_reg, headptr_val); 131 mb_chann->i2x_head = headptr_val; 132 } 133 134 static inline void 135 mailbox_set_tailptr(struct mailbox_channel *mb_chann, u32 tailptr_val) 136 { 137 mailbox_reg_write(mb_chann, mb_chann->res[CHAN_RES_X2I].mb_tail_ptr_reg, tailptr_val); 138 mb_chann->x2i_tail = tailptr_val; 139 } 140 141 static inline u32 142 mailbox_get_headptr(struct mailbox_channel *mb_chann, enum channel_res_type type) 143 { 144 return mailbox_reg_read(mb_chann, mb_chann->res[type].mb_head_ptr_reg); 145 } 146 147 static inline u32 148 mailbox_get_tailptr(struct mailbox_channel *mb_chann, enum channel_res_type type) 149 { 150 return mailbox_reg_read(mb_chann, mb_chann->res[type].mb_tail_ptr_reg); 151 } 152 153 static inline u32 154 mailbox_get_ringbuf_size(struct mailbox_channel *mb_chann, enum channel_res_type type) 155 { 156 return mb_chann->res[type].rb_size; 157 } 158 159 static inline int mailbox_validate_msgid(int msg_id) 160 { 161 return (msg_id & MAGIC_VAL_MASK) == MAGIC_VAL; 162 } 163 164 static int mailbox_acquire_msgid(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) 165 { 166 u32 msg_id; 167 int ret; 168 169 ret = xa_alloc_cyclic_irq(&mb_chann->chan_xa, &msg_id, mb_msg, 170 XA_LIMIT(0, MAX_MSG_ID_ENTRIES - 1), 171 &mb_chann->next_msgid, GFP_NOWAIT); 172 if (ret < 0) 173 return ret; 174 175 /* 176 * Add MAGIC_VAL to the higher bits. 177 */ 178 msg_id |= MAGIC_VAL; 179 return msg_id; 180 } 181 182 static void mailbox_release_msgid(struct mailbox_channel *mb_chann, int msg_id) 183 { 184 msg_id &= ~MAGIC_VAL_MASK; 185 xa_erase_irq(&mb_chann->chan_xa, msg_id); 186 } 187 188 static void mailbox_release_msg(struct mailbox_channel *mb_chann, 189 struct mailbox_msg *mb_msg) 190 { 191 MB_DBG(mb_chann, "msg_id 0x%x msg opcode 0x%x", 192 mb_msg->pkg.header.id, mb_msg->pkg.header.opcode); 193 if (mb_msg->notify_cb) 194 mb_msg->notify_cb(mb_msg->handle, NULL, 0); 195 kfree(mb_msg); 196 } 197 198 static int 199 mailbox_send_msg(struct mailbox_channel *mb_chann, struct mailbox_msg *mb_msg) 200 { 201 void __iomem *write_addr; 202 u32 ringbuf_size; 203 u32 head, tail; 204 u32 start_addr; 205 u32 tmp_tail; 206 int ret; 207 208 head = mailbox_get_headptr(mb_chann, CHAN_RES_X2I); 209 tail = mb_chann->x2i_tail; 210 ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I) - sizeof(u32); 211 start_addr = mb_chann->res[CHAN_RES_X2I].rb_start_addr; 212 tmp_tail = tail + mb_msg->pkg_size; 213 214 check_again: 215 if (tail >= head && tmp_tail > ringbuf_size) { 216 write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; 217 writel(TOMBSTONE, write_addr); 218 219 /* tombstone is set. Write from the start of the ringbuf */ 220 tail = 0; 221 tmp_tail = tail + mb_msg->pkg_size; 222 } 223 224 if (tail < head && tmp_tail >= head) { 225 ret = read_poll_timeout(mailbox_get_headptr, head, 226 tmp_tail < head || tail >= head, 227 1, 100, false, mb_chann, CHAN_RES_X2I); 228 if (ret) 229 return ret; 230 231 if (tail >= head) 232 goto check_again; 233 } 234 235 write_addr = mb_chann->mb->res.ringbuf_base + start_addr + tail; 236 memcpy_toio(write_addr, &mb_msg->pkg, mb_msg->pkg_size); 237 mailbox_set_tailptr(mb_chann, tail + mb_msg->pkg_size); 238 239 trace_mbox_set_tail(MAILBOX_NAME, mb_chann->msix_irq, 240 mb_msg->pkg.header.opcode, 241 mb_msg->pkg.header.id); 242 243 return 0; 244 } 245 246 static int 247 mailbox_get_resp(struct mailbox_channel *mb_chann, struct xdna_msg_header *header, 248 void __iomem *data) 249 { 250 struct mailbox_msg *mb_msg; 251 int msg_id; 252 int ret = 0; 253 254 msg_id = header->id; 255 if (!mailbox_validate_msgid(msg_id)) { 256 MB_ERR(mb_chann, "Bad message ID 0x%x", msg_id); 257 return -EINVAL; 258 } 259 260 msg_id &= ~MAGIC_VAL_MASK; 261 mb_msg = xa_erase_irq(&mb_chann->chan_xa, msg_id); 262 if (!mb_msg) { 263 MB_ERR(mb_chann, "Cannot find msg 0x%x", msg_id); 264 return -EINVAL; 265 } 266 267 MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x", 268 header->opcode, header->total_size, header->id); 269 if (mb_msg->notify_cb) { 270 ret = mb_msg->notify_cb(mb_msg->handle, data, header->total_size); 271 if (unlikely(ret)) 272 MB_ERR(mb_chann, "Message callback ret %d", ret); 273 } 274 275 kfree(mb_msg); 276 return ret; 277 } 278 279 static int mailbox_get_msg(struct mailbox_channel *mb_chann) 280 { 281 struct xdna_msg_header header; 282 void __iomem *read_addr; 283 u32 msg_size, rest; 284 u32 ringbuf_size; 285 u32 head, tail; 286 u32 start_addr; 287 int ret; 288 289 tail = mailbox_get_tailptr(mb_chann, CHAN_RES_I2X); 290 head = mb_chann->i2x_head; 291 ringbuf_size = mailbox_get_ringbuf_size(mb_chann, CHAN_RES_I2X); 292 start_addr = mb_chann->res[CHAN_RES_I2X].rb_start_addr; 293 294 if (unlikely(tail > ringbuf_size || !IS_ALIGNED(tail, 4))) { 295 MB_WARN_ONCE(mb_chann, "Invalid tail 0x%x", tail); 296 return -EINVAL; 297 } 298 299 /* ringbuf empty */ 300 if (head == tail) 301 return -ENOENT; 302 303 if (head == ringbuf_size) 304 head = 0; 305 306 /* Peek size of the message or TOMBSTONE */ 307 read_addr = mb_chann->mb->res.ringbuf_base + start_addr + head; 308 header.total_size = readl(read_addr); 309 /* size is TOMBSTONE, set next read from 0 */ 310 if (header.total_size == TOMBSTONE) { 311 if (head < tail) { 312 MB_WARN_ONCE(mb_chann, "Tombstone, head 0x%x tail 0x%x", 313 head, tail); 314 return -EINVAL; 315 } 316 mailbox_set_headptr(mb_chann, 0); 317 return 0; 318 } 319 320 if (unlikely(!header.total_size || !IS_ALIGNED(header.total_size, 4))) { 321 MB_WARN_ONCE(mb_chann, "Invalid total size 0x%x", header.total_size); 322 return -EINVAL; 323 } 324 msg_size = sizeof(header) + header.total_size; 325 326 if (msg_size > ringbuf_size - head || msg_size > tail - head) { 327 MB_WARN_ONCE(mb_chann, "Invalid message size %d, tail %d, head %d", 328 msg_size, tail, head); 329 return -EINVAL; 330 } 331 332 rest = sizeof(header) - sizeof(u32); 333 read_addr += sizeof(u32); 334 memcpy_fromio((u32 *)&header + 1, read_addr, rest); 335 read_addr += rest; 336 337 ret = mailbox_get_resp(mb_chann, &header, read_addr); 338 339 mailbox_set_headptr(mb_chann, head + msg_size); 340 /* After update head, it can equal to ringbuf_size. This is expected. */ 341 trace_mbox_set_head(MAILBOX_NAME, mb_chann->msix_irq, 342 header.opcode, header.id); 343 344 return ret; 345 } 346 347 static irqreturn_t mailbox_irq_handler(int irq, void *p) 348 { 349 struct mailbox_channel *mb_chann = p; 350 351 trace_mbox_irq_handle(MAILBOX_NAME, irq); 352 /* Schedule a rx_work to call the callback functions */ 353 queue_work(mb_chann->work_q, &mb_chann->rx_work); 354 355 return IRQ_HANDLED; 356 } 357 358 static void mailbox_rx_worker(struct work_struct *rx_work) 359 { 360 struct mailbox_channel *mb_chann; 361 int ret; 362 363 mb_chann = container_of(rx_work, struct mailbox_channel, rx_work); 364 trace_mbox_rx_worker(MAILBOX_NAME, mb_chann->msix_irq); 365 366 if (READ_ONCE(mb_chann->bad_state)) { 367 MB_ERR(mb_chann, "Channel in bad state, work aborted"); 368 return; 369 } 370 371 again: 372 mailbox_irq_acknowledge(mb_chann); 373 374 while (1) { 375 /* 376 * If return is 0, keep consuming next message, until there is 377 * no messages or an error happened. 378 */ 379 ret = mailbox_get_msg(mb_chann); 380 if (ret == -ENOENT) 381 break; 382 383 /* Other error means device doesn't look good, disable irq. */ 384 if (unlikely(ret)) { 385 MB_ERR(mb_chann, "Unexpected ret %d, disable irq", ret); 386 WRITE_ONCE(mb_chann->bad_state, true); 387 return; 388 } 389 } 390 391 /* 392 * The hardware will not generate interrupt if firmware creates a new 393 * response right after driver clears interrupt register. Check 394 * the interrupt register to make sure there is not any new response 395 * before exiting. 396 */ 397 if (mailbox_irq_status(mb_chann)) 398 goto again; 399 } 400 401 int xdna_mailbox_send_msg(struct mailbox_channel *mb_chann, 402 const struct xdna_mailbox_msg *msg, u64 tx_timeout) 403 { 404 struct xdna_msg_header *header; 405 struct mailbox_msg *mb_msg; 406 size_t pkg_size; 407 int ret; 408 409 pkg_size = sizeof(*header) + msg->send_size; 410 if (pkg_size > mailbox_get_ringbuf_size(mb_chann, CHAN_RES_X2I)) { 411 MB_ERR(mb_chann, "Message size larger than ringbuf size"); 412 return -EINVAL; 413 } 414 415 if (unlikely(!IS_ALIGNED(msg->send_size, 4))) { 416 MB_ERR(mb_chann, "Message must be 4 bytes align"); 417 return -EINVAL; 418 } 419 420 /* The fist word in payload can NOT be TOMBSTONE */ 421 if (unlikely(((u32 *)msg->send_data)[0] == TOMBSTONE)) { 422 MB_ERR(mb_chann, "Tomb stone in data"); 423 return -EINVAL; 424 } 425 426 if (READ_ONCE(mb_chann->bad_state)) { 427 MB_ERR(mb_chann, "Channel in bad state"); 428 return -EPIPE; 429 } 430 431 mb_msg = kzalloc(sizeof(*mb_msg) + pkg_size, GFP_KERNEL); 432 if (!mb_msg) 433 return -ENOMEM; 434 435 mb_msg->handle = msg->handle; 436 mb_msg->notify_cb = msg->notify_cb; 437 mb_msg->pkg_size = pkg_size; 438 439 header = &mb_msg->pkg.header; 440 /* 441 * Hardware use total_size and size to split huge message. 442 * We do not support it here. Thus the values are the same. 443 */ 444 header->total_size = msg->send_size; 445 header->sz_ver = FIELD_PREP(MSG_BODY_SZ, msg->send_size) | 446 FIELD_PREP(MSG_PROTO_VER, MSG_PROTOCOL_VERSION); 447 header->opcode = msg->opcode; 448 memcpy(mb_msg->pkg.payload, msg->send_data, msg->send_size); 449 450 ret = mailbox_acquire_msgid(mb_chann, mb_msg); 451 if (unlikely(ret < 0)) { 452 MB_ERR(mb_chann, "mailbox_acquire_msgid failed"); 453 goto msg_id_failed; 454 } 455 header->id = ret; 456 457 MB_DBG(mb_chann, "opcode 0x%x size %d id 0x%x", 458 header->opcode, header->total_size, header->id); 459 460 ret = mailbox_send_msg(mb_chann, mb_msg); 461 if (ret) { 462 MB_DBG(mb_chann, "Error in mailbox send msg, ret %d", ret); 463 goto release_id; 464 } 465 466 return 0; 467 468 release_id: 469 mailbox_release_msgid(mb_chann, header->id); 470 msg_id_failed: 471 kfree(mb_msg); 472 return ret; 473 } 474 475 struct mailbox_channel *xdna_mailbox_alloc_channel(struct mailbox *mb) 476 { 477 struct mailbox_channel *mb_chann; 478 479 mb_chann = kzalloc_obj(*mb_chann); 480 if (!mb_chann) 481 return NULL; 482 483 INIT_WORK(&mb_chann->rx_work, mailbox_rx_worker); 484 mb_chann->work_q = create_singlethread_workqueue(MAILBOX_NAME); 485 if (!mb_chann->work_q) { 486 MB_ERR(mb_chann, "Create workqueue failed"); 487 goto free_chann; 488 } 489 mb_chann->mb = mb; 490 491 return mb_chann; 492 493 free_chann: 494 kfree(mb_chann); 495 return NULL; 496 } 497 498 void xdna_mailbox_free_channel(struct mailbox_channel *mb_chann) 499 { 500 if (!mb_chann) 501 return; 502 503 destroy_workqueue(mb_chann->work_q); 504 kfree(mb_chann); 505 } 506 507 int 508 xdna_mailbox_start_channel(struct mailbox_channel *mb_chann, 509 const struct xdna_mailbox_chann_res *x2i, 510 const struct xdna_mailbox_chann_res *i2x, 511 u32 iohub_int_addr, 512 int mb_irq) 513 { 514 int ret; 515 516 if (!is_power_of_2(x2i->rb_size) || !is_power_of_2(i2x->rb_size)) { 517 pr_err("Ring buf size must be power of 2\n"); 518 return -EINVAL; 519 } 520 521 mb_chann->msix_irq = mb_irq; 522 mb_chann->iohub_int_addr = iohub_int_addr; 523 memcpy(&mb_chann->res[CHAN_RES_X2I], x2i, sizeof(*x2i)); 524 memcpy(&mb_chann->res[CHAN_RES_I2X], i2x, sizeof(*i2x)); 525 526 xa_init_flags(&mb_chann->chan_xa, XA_FLAGS_ALLOC | XA_FLAGS_LOCK_IRQ); 527 mb_chann->x2i_tail = mailbox_get_tailptr(mb_chann, CHAN_RES_X2I); 528 mb_chann->i2x_head = mailbox_get_headptr(mb_chann, CHAN_RES_I2X); 529 530 /* Everything look good. Time to enable irq handler */ 531 ret = request_irq(mb_irq, mailbox_irq_handler, 0, MAILBOX_NAME, mb_chann); 532 if (ret) { 533 MB_ERR(mb_chann, "Failed to request irq %d ret %d", mb_irq, ret); 534 return ret; 535 } 536 537 mb_chann->bad_state = false; 538 mailbox_irq_acknowledge(mb_chann); 539 540 MB_DBG(mb_chann, "Mailbox channel started (irq: %d)", mb_chann->msix_irq); 541 return 0; 542 } 543 544 void xdna_mailbox_stop_channel(struct mailbox_channel *mb_chann) 545 { 546 struct mailbox_msg *mb_msg; 547 unsigned long msg_id; 548 549 if (!mb_chann) 550 return; 551 552 /* Disable an irq and wait. This might sleep. */ 553 free_irq(mb_chann->msix_irq, mb_chann); 554 555 /* Cancel RX work and wait for it to finish */ 556 drain_workqueue(mb_chann->work_q); 557 558 /* We can clean up and release resources */ 559 xa_for_each_start(&mb_chann->chan_xa, msg_id, mb_msg, mb_chann->next_msgid) 560 mailbox_release_msg(mb_chann, mb_msg); 561 xa_for_each_range(&mb_chann->chan_xa, msg_id, mb_msg, 0, mb_chann->next_msgid - 1) 562 mailbox_release_msg(mb_chann, mb_msg); 563 xa_destroy(&mb_chann->chan_xa); 564 565 MB_DBG(mb_chann, "Mailbox channel stopped, irq: %d", mb_chann->msix_irq); 566 } 567 568 struct mailbox *xdnam_mailbox_create(struct drm_device *ddev, 569 const struct xdna_mailbox_res *res) 570 { 571 struct mailbox *mb; 572 573 mb = drmm_kzalloc(ddev, sizeof(*mb), GFP_KERNEL); 574 if (!mb) 575 return NULL; 576 mb->dev = ddev->dev; 577 578 /* mailbox and ring buf base and size information */ 579 memcpy(&mb->res, res, sizeof(*res)); 580 581 return mb; 582 } 583