1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (C) 2017 Broadcom 3 4 /* 5 * Broadcom FlexRM Mailbox Driver 6 * 7 * Each Broadcom FlexSparx4 offload engine is implemented as an 8 * extension to Broadcom FlexRM ring manager. The FlexRM ring 9 * manager provides a set of rings which can be used to submit 10 * work to a FlexSparx4 offload engine. 11 * 12 * This driver creates a mailbox controller using a set of FlexRM 13 * rings where each mailbox channel represents a separate FlexRM ring. 14 */ 15 16 #include <asm/barrier.h> 17 #include <asm/byteorder.h> 18 #include <linux/atomic.h> 19 #include <linux/bitmap.h> 20 #include <linux/debugfs.h> 21 #include <linux/delay.h> 22 #include <linux/device.h> 23 #include <linux/dma-mapping.h> 24 #include <linux/dmapool.h> 25 #include <linux/err.h> 26 #include <linux/interrupt.h> 27 #include <linux/kernel.h> 28 #include <linux/mailbox_controller.h> 29 #include <linux/mailbox_client.h> 30 #include <linux/mailbox/brcm-message.h> 31 #include <linux/module.h> 32 #include <linux/msi.h> 33 #include <linux/of_address.h> 34 #include <linux/of_irq.h> 35 #include <linux/platform_device.h> 36 #include <linux/spinlock.h> 37 38 /* ====== FlexRM register defines ===== */ 39 40 /* FlexRM configuration */ 41 #define RING_REGS_SIZE 0x10000 42 #define RING_DESC_SIZE 8 43 #define RING_DESC_INDEX(offset) \ 44 ((offset) / RING_DESC_SIZE) 45 #define RING_DESC_OFFSET(index) \ 46 ((index) * RING_DESC_SIZE) 47 #define RING_MAX_REQ_COUNT 1024 48 #define RING_BD_ALIGN_ORDER 12 49 #define RING_BD_ALIGN_CHECK(addr) \ 50 (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1))) 51 #define RING_BD_TOGGLE_INVALID(offset) \ 52 (((offset) >> RING_BD_ALIGN_ORDER) & 0x1) 53 #define RING_BD_TOGGLE_VALID(offset) \ 54 (!RING_BD_TOGGLE_INVALID(offset)) 55 #define RING_BD_DESC_PER_REQ 32 56 #define RING_BD_DESC_COUNT \ 57 (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ) 58 #define RING_BD_SIZE \ 59 (RING_BD_DESC_COUNT * RING_DESC_SIZE) 60 #define RING_CMPL_ALIGN_ORDER 13 61 #define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT 62 #define RING_CMPL_SIZE \ 63 (RING_CMPL_DESC_COUNT * RING_DESC_SIZE) 64 #define RING_VER_MAGIC 0x76303031 65 66 /* Per-Ring register offsets */ 67 #define RING_VER 0x000 68 #define RING_BD_START_ADDR 0x004 69 #define RING_BD_READ_PTR 0x008 70 #define RING_BD_WRITE_PTR 0x00c 71 #define RING_BD_READ_PTR_DDR_LS 0x010 72 #define RING_BD_READ_PTR_DDR_MS 0x014 73 #define RING_CMPL_START_ADDR 0x018 74 #define RING_CMPL_WRITE_PTR 0x01c 75 #define RING_NUM_REQ_RECV_LS 0x020 76 #define RING_NUM_REQ_RECV_MS 0x024 77 #define RING_NUM_REQ_TRANS_LS 0x028 78 #define RING_NUM_REQ_TRANS_MS 0x02c 79 #define RING_NUM_REQ_OUTSTAND 0x030 80 #define RING_CONTROL 0x034 81 #define RING_FLUSH_DONE 0x038 82 #define RING_MSI_ADDR_LS 0x03c 83 #define RING_MSI_ADDR_MS 0x040 84 #define RING_MSI_CONTROL 0x048 85 #define RING_BD_READ_PTR_DDR_CONTROL 0x04c 86 #define RING_MSI_DATA_VALUE 0x064 87 88 /* Register RING_BD_START_ADDR fields */ 89 #define BD_LAST_UPDATE_HW_SHIFT 28 90 #define BD_LAST_UPDATE_HW_MASK 0x1 91 #define BD_START_ADDR_VALUE(pa) \ 92 ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff)) 93 #define BD_START_ADDR_DECODE(val) \ 94 ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER) 95 96 /* Register RING_CMPL_START_ADDR fields */ 97 #define CMPL_START_ADDR_VALUE(pa) \ 98 ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff)) 99 100 /* Register RING_CONTROL fields */ 101 #define CONTROL_MASK_DISABLE_CONTROL 12 102 #define CONTROL_FLUSH_SHIFT 5 103 #define CONTROL_ACTIVE_SHIFT 4 104 #define CONTROL_RATE_ADAPT_MASK 0xf 105 #define CONTROL_RATE_DYNAMIC 0x0 106 #define CONTROL_RATE_FAST 0x8 107 #define CONTROL_RATE_MEDIUM 0x9 108 #define CONTROL_RATE_SLOW 0xa 109 #define CONTROL_RATE_IDLE 0xb 110 111 /* Register RING_FLUSH_DONE fields */ 112 #define FLUSH_DONE_MASK 0x1 113 114 /* Register RING_MSI_CONTROL fields */ 115 #define MSI_TIMER_VAL_SHIFT 16 116 #define MSI_TIMER_VAL_MASK 0xffff 117 #define MSI_ENABLE_SHIFT 15 118 #define MSI_ENABLE_MASK 0x1 119 #define MSI_COUNT_SHIFT 0 120 #define MSI_COUNT_MASK 0x3ff 121 122 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */ 123 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16 124 #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff 125 #define BD_READ_PTR_DDR_ENABLE_SHIFT 15 126 #define BD_READ_PTR_DDR_ENABLE_MASK 0x1 127 128 /* ====== FlexRM ring descriptor defines ===== */ 129 130 /* Completion descriptor format */ 131 #define CMPL_OPAQUE_SHIFT 0 132 #define CMPL_OPAQUE_MASK 0xffff 133 #define CMPL_ENGINE_STATUS_SHIFT 16 134 #define CMPL_ENGINE_STATUS_MASK 0xffff 135 #define CMPL_DME_STATUS_SHIFT 32 136 #define CMPL_DME_STATUS_MASK 0xffff 137 #define CMPL_RM_STATUS_SHIFT 48 138 #define CMPL_RM_STATUS_MASK 0xffff 139 140 /* Completion DME status code */ 141 #define DME_STATUS_MEM_COR_ERR BIT(0) 142 #define DME_STATUS_MEM_UCOR_ERR BIT(1) 143 #define DME_STATUS_FIFO_UNDERFLOW BIT(2) 144 #define DME_STATUS_FIFO_OVERFLOW BIT(3) 145 #define DME_STATUS_RRESP_ERR BIT(4) 146 #define DME_STATUS_BRESP_ERR BIT(5) 147 #define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \ 148 DME_STATUS_MEM_UCOR_ERR | \ 149 DME_STATUS_FIFO_UNDERFLOW | \ 150 DME_STATUS_FIFO_OVERFLOW | \ 151 DME_STATUS_RRESP_ERR | \ 152 DME_STATUS_BRESP_ERR) 153 154 /* Completion RM status code */ 155 #define RM_STATUS_CODE_SHIFT 0 156 #define RM_STATUS_CODE_MASK 0x3ff 157 #define RM_STATUS_CODE_GOOD 0x0 158 #define RM_STATUS_CODE_AE_TIMEOUT 0x3ff 159 160 /* General descriptor format */ 161 #define DESC_TYPE_SHIFT 60 162 #define DESC_TYPE_MASK 0xf 163 #define DESC_PAYLOAD_SHIFT 0 164 #define DESC_PAYLOAD_MASK 0x0fffffffffffffff 165 166 /* Null descriptor format */ 167 #define NULL_TYPE 0 168 #define NULL_TOGGLE_SHIFT 58 169 #define NULL_TOGGLE_MASK 0x1 170 171 /* Header descriptor format */ 172 #define HEADER_TYPE 1 173 #define HEADER_TOGGLE_SHIFT 58 174 #define HEADER_TOGGLE_MASK 0x1 175 #define HEADER_ENDPKT_SHIFT 57 176 #define HEADER_ENDPKT_MASK 0x1 177 #define HEADER_STARTPKT_SHIFT 56 178 #define HEADER_STARTPKT_MASK 0x1 179 #define HEADER_BDCOUNT_SHIFT 36 180 #define HEADER_BDCOUNT_MASK 0x1f 181 #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK 182 #define HEADER_FLAGS_SHIFT 16 183 #define HEADER_FLAGS_MASK 0xffff 184 #define HEADER_OPAQUE_SHIFT 0 185 #define HEADER_OPAQUE_MASK 0xffff 186 187 /* Source (SRC) descriptor format */ 188 #define SRC_TYPE 2 189 #define SRC_LENGTH_SHIFT 44 190 #define SRC_LENGTH_MASK 0xffff 191 #define SRC_ADDR_SHIFT 0 192 #define SRC_ADDR_MASK 0x00000fffffffffff 193 194 /* Destination (DST) descriptor format */ 195 #define DST_TYPE 3 196 #define DST_LENGTH_SHIFT 44 197 #define DST_LENGTH_MASK 0xffff 198 #define DST_ADDR_SHIFT 0 199 #define DST_ADDR_MASK 0x00000fffffffffff 200 201 /* Immediate (IMM) descriptor format */ 202 #define IMM_TYPE 4 203 #define IMM_DATA_SHIFT 0 204 #define IMM_DATA_MASK 0x0fffffffffffffff 205 206 /* Next pointer (NPTR) descriptor format */ 207 #define NPTR_TYPE 5 208 #define NPTR_TOGGLE_SHIFT 58 209 #define NPTR_TOGGLE_MASK 0x1 210 #define NPTR_ADDR_SHIFT 0 211 #define NPTR_ADDR_MASK 0x00000fffffffffff 212 213 /* Mega source (MSRC) descriptor format */ 214 #define MSRC_TYPE 6 215 #define MSRC_LENGTH_SHIFT 44 216 #define MSRC_LENGTH_MASK 0xffff 217 #define MSRC_ADDR_SHIFT 0 218 #define MSRC_ADDR_MASK 0x00000fffffffffff 219 220 /* Mega destination (MDST) descriptor format */ 221 #define MDST_TYPE 7 222 #define MDST_LENGTH_SHIFT 44 223 #define MDST_LENGTH_MASK 0xffff 224 #define MDST_ADDR_SHIFT 0 225 #define MDST_ADDR_MASK 0x00000fffffffffff 226 227 /* Source with tlast (SRCT) descriptor format */ 228 #define SRCT_TYPE 8 229 #define SRCT_LENGTH_SHIFT 44 230 #define SRCT_LENGTH_MASK 0xffff 231 #define SRCT_ADDR_SHIFT 0 232 #define SRCT_ADDR_MASK 0x00000fffffffffff 233 234 /* Destination with tlast (DSTT) descriptor format */ 235 #define DSTT_TYPE 9 236 #define DSTT_LENGTH_SHIFT 44 237 #define DSTT_LENGTH_MASK 0xffff 238 #define DSTT_ADDR_SHIFT 0 239 #define DSTT_ADDR_MASK 0x00000fffffffffff 240 241 /* Immediate with tlast (IMMT) descriptor format */ 242 #define IMMT_TYPE 10 243 #define IMMT_DATA_SHIFT 0 244 #define IMMT_DATA_MASK 0x0fffffffffffffff 245 246 /* Descriptor helper macros */ 247 #define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m)) 248 #define DESC_ENC(_d, _v, _s, _m) \ 249 do { \ 250 (_d) &= ~((u64)(_m) << (_s)); \ 251 (_d) |= (((u64)(_v) & (_m)) << (_s)); \ 252 } while (0) 253 254 /* ====== FlexRM data structures ===== */ 255 256 struct flexrm_ring { 257 /* Unprotected members */ 258 int num; 259 struct flexrm_mbox *mbox; 260 void __iomem *regs; 261 bool irq_requested; 262 unsigned int irq; 263 cpumask_t irq_aff_hint; 264 unsigned int msi_timer_val; 265 unsigned int msi_count_threshold; 266 struct brcm_message *requests[RING_MAX_REQ_COUNT]; 267 void *bd_base; 268 dma_addr_t bd_dma_base; 269 u32 bd_write_offset; 270 void *cmpl_base; 271 dma_addr_t cmpl_dma_base; 272 /* Atomic stats */ 273 atomic_t msg_send_count; 274 atomic_t msg_cmpl_count; 275 /* Protected members */ 276 spinlock_t lock; 277 DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT); 278 u32 cmpl_read_offset; 279 }; 280 281 struct flexrm_mbox { 282 struct device *dev; 283 void __iomem *regs; 284 u32 num_rings; 285 struct flexrm_ring *rings; 286 struct dma_pool *bd_pool; 287 struct dma_pool *cmpl_pool; 288 struct dentry *root; 289 struct mbox_controller controller; 290 }; 291 292 /* ====== FlexRM ring descriptor helper routines ===== */ 293 294 static u64 flexrm_read_desc(void *desc_ptr) 295 { 296 return le64_to_cpu(*((u64 *)desc_ptr)); 297 } 298 299 static void flexrm_write_desc(void *desc_ptr, u64 desc) 300 { 301 *((u64 *)desc_ptr) = cpu_to_le64(desc); 302 } 303 304 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc) 305 { 306 return (u32)(cmpl_desc & CMPL_OPAQUE_MASK); 307 } 308 309 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc) 310 { 311 u32 status; 312 313 status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT, 314 CMPL_DME_STATUS_MASK); 315 if (status & DME_STATUS_ERROR_MASK) 316 return -EIO; 317 318 status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT, 319 CMPL_RM_STATUS_MASK); 320 status &= RM_STATUS_CODE_MASK; 321 if (status == RM_STATUS_CODE_AE_TIMEOUT) 322 return -ETIMEDOUT; 323 324 return 0; 325 } 326 327 static bool flexrm_is_next_table_desc(void *desc_ptr) 328 { 329 u64 desc = flexrm_read_desc(desc_ptr); 330 u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 331 332 return (type == NPTR_TYPE) ? true : false; 333 } 334 335 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr) 336 { 337 u64 desc = 0; 338 339 DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 340 DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK); 341 DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK); 342 343 return desc; 344 } 345 346 static u64 flexrm_null_desc(u32 toggle) 347 { 348 u64 desc = 0; 349 350 DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 351 DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK); 352 353 return desc; 354 } 355 356 static u32 flexrm_estimate_header_desc_count(u32 nhcnt) 357 { 358 u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX; 359 360 if (!(nhcnt % HEADER_BDCOUNT_MAX)) 361 hcnt += 1; 362 363 return hcnt; 364 } 365 366 static void flexrm_flip_header_toggle(void *desc_ptr) 367 { 368 u64 desc = flexrm_read_desc(desc_ptr); 369 370 if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT)) 371 desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT); 372 else 373 desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT); 374 375 flexrm_write_desc(desc_ptr, desc); 376 } 377 378 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt, 379 u32 bdcount, u32 flags, u32 opaque) 380 { 381 u64 desc = 0; 382 383 DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 384 DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK); 385 DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK); 386 DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK); 387 DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK); 388 DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK); 389 DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK); 390 391 return desc; 392 } 393 394 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid, 395 u64 desc, void **desc_ptr, u32 *toggle, 396 void *start_desc, void *end_desc) 397 { 398 u64 d; 399 u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount; 400 401 /* Sanity check */ 402 if (nhcnt <= nhpos) 403 return; 404 405 /* 406 * Each request or packet start with a HEADER descriptor followed 407 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST, 408 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors 409 * following a HEADER descriptor is represented by BDCOUNT field 410 * of HEADER descriptor. The max value of BDCOUNT field is 31 which 411 * means we can only have 31 non-HEADER descriptors following one 412 * HEADER descriptor. 413 * 414 * In general use, number of non-HEADER descriptors can easily go 415 * beyond 31. To tackle this situation, we have packet (or request) 416 * extension bits (STARTPKT and ENDPKT) in the HEADER descriptor. 417 * 418 * To use packet extension, the first HEADER descriptor of request 419 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate 420 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last 421 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the 422 * TOGGLE bit of the first HEADER will be set to invalid state to 423 * ensure that FlexRM does not start fetching descriptors till all 424 * descriptors are enqueued. The user of this function will flip 425 * the TOGGLE bit of first HEADER after all descriptors are 426 * enqueued. 427 */ 428 429 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) { 430 /* Prepare the header descriptor */ 431 nhavail = (nhcnt - nhpos); 432 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle); 433 _startpkt = (nhpos == 0) ? 0x1 : 0x0; 434 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0; 435 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ? 436 nhavail : HEADER_BDCOUNT_MAX; 437 if (nhavail <= HEADER_BDCOUNT_MAX) 438 _bdcount = nhavail; 439 else 440 _bdcount = HEADER_BDCOUNT_MAX; 441 d = flexrm_header_desc(_toggle, _startpkt, _endpkt, 442 _bdcount, 0x0, reqid); 443 444 /* Write header descriptor */ 445 flexrm_write_desc(*desc_ptr, d); 446 447 /* Point to next descriptor */ 448 *desc_ptr += sizeof(desc); 449 if (*desc_ptr == end_desc) 450 *desc_ptr = start_desc; 451 452 /* Skip next pointer descriptors */ 453 while (flexrm_is_next_table_desc(*desc_ptr)) { 454 *toggle = (*toggle) ? 0 : 1; 455 *desc_ptr += sizeof(desc); 456 if (*desc_ptr == end_desc) 457 *desc_ptr = start_desc; 458 } 459 } 460 461 /* Write desired descriptor */ 462 flexrm_write_desc(*desc_ptr, desc); 463 464 /* Point to next descriptor */ 465 *desc_ptr += sizeof(desc); 466 if (*desc_ptr == end_desc) 467 *desc_ptr = start_desc; 468 469 /* Skip next pointer descriptors */ 470 while (flexrm_is_next_table_desc(*desc_ptr)) { 471 *toggle = (*toggle) ? 0 : 1; 472 *desc_ptr += sizeof(desc); 473 if (*desc_ptr == end_desc) 474 *desc_ptr = start_desc; 475 } 476 } 477 478 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length) 479 { 480 u64 desc = 0; 481 482 DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 483 DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK); 484 DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK); 485 486 return desc; 487 } 488 489 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16) 490 { 491 u64 desc = 0; 492 493 DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 494 DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK); 495 DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK); 496 497 return desc; 498 } 499 500 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length) 501 { 502 u64 desc = 0; 503 504 DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 505 DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK); 506 DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK); 507 508 return desc; 509 } 510 511 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16) 512 { 513 u64 desc = 0; 514 515 DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 516 DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK); 517 DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK); 518 519 return desc; 520 } 521 522 static u64 flexrm_imm_desc(u64 data) 523 { 524 u64 desc = 0; 525 526 DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 527 DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK); 528 529 return desc; 530 } 531 532 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length) 533 { 534 u64 desc = 0; 535 536 DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 537 DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK); 538 DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK); 539 540 return desc; 541 } 542 543 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length) 544 { 545 u64 desc = 0; 546 547 DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 548 DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK); 549 DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK); 550 551 return desc; 552 } 553 554 static u64 flexrm_immt_desc(u64 data) 555 { 556 u64 desc = 0; 557 558 DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 559 DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK); 560 561 return desc; 562 } 563 564 static bool flexrm_spu_sanity_check(struct brcm_message *msg) 565 { 566 struct scatterlist *sg; 567 568 if (!msg->spu.src || !msg->spu.dst) 569 return false; 570 for (sg = msg->spu.src; sg; sg = sg_next(sg)) { 571 if (sg->length & 0xf) { 572 if (sg->length > SRC_LENGTH_MASK) 573 return false; 574 } else { 575 if (sg->length > (MSRC_LENGTH_MASK * 16)) 576 return false; 577 } 578 } 579 for (sg = msg->spu.dst; sg; sg = sg_next(sg)) { 580 if (sg->length & 0xf) { 581 if (sg->length > DST_LENGTH_MASK) 582 return false; 583 } else { 584 if (sg->length > (MDST_LENGTH_MASK * 16)) 585 return false; 586 } 587 } 588 589 return true; 590 } 591 592 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg) 593 { 594 u32 cnt = 0; 595 unsigned int dst_target = 0; 596 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst; 597 598 while (src_sg || dst_sg) { 599 if (src_sg) { 600 cnt++; 601 dst_target = src_sg->length; 602 src_sg = sg_next(src_sg); 603 } else 604 dst_target = UINT_MAX; 605 606 while (dst_target && dst_sg) { 607 cnt++; 608 if (dst_sg->length < dst_target) 609 dst_target -= dst_sg->length; 610 else 611 dst_target = 0; 612 dst_sg = sg_next(dst_sg); 613 } 614 } 615 616 return cnt; 617 } 618 619 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg) 620 { 621 int rc; 622 623 rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 624 DMA_TO_DEVICE); 625 if (!rc) 626 return -EIO; 627 628 rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), 629 DMA_FROM_DEVICE); 630 if (!rc) { 631 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 632 DMA_TO_DEVICE); 633 return -EIO; 634 } 635 636 return 0; 637 } 638 639 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg) 640 { 641 dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), 642 DMA_FROM_DEVICE); 643 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 644 DMA_TO_DEVICE); 645 } 646 647 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt, 648 u32 reqid, void *desc_ptr, u32 toggle, 649 void *start_desc, void *end_desc) 650 { 651 u64 d; 652 u32 nhpos = 0; 653 void *orig_desc_ptr = desc_ptr; 654 unsigned int dst_target = 0; 655 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst; 656 657 while (src_sg || dst_sg) { 658 if (src_sg) { 659 if (sg_dma_len(src_sg) & 0xf) 660 d = flexrm_src_desc(sg_dma_address(src_sg), 661 sg_dma_len(src_sg)); 662 else 663 d = flexrm_msrc_desc(sg_dma_address(src_sg), 664 sg_dma_len(src_sg)/16); 665 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 666 d, &desc_ptr, &toggle, 667 start_desc, end_desc); 668 nhpos++; 669 dst_target = sg_dma_len(src_sg); 670 src_sg = sg_next(src_sg); 671 } else 672 dst_target = UINT_MAX; 673 674 while (dst_target && dst_sg) { 675 if (sg_dma_len(dst_sg) & 0xf) 676 d = flexrm_dst_desc(sg_dma_address(dst_sg), 677 sg_dma_len(dst_sg)); 678 else 679 d = flexrm_mdst_desc(sg_dma_address(dst_sg), 680 sg_dma_len(dst_sg)/16); 681 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 682 d, &desc_ptr, &toggle, 683 start_desc, end_desc); 684 nhpos++; 685 if (sg_dma_len(dst_sg) < dst_target) 686 dst_target -= sg_dma_len(dst_sg); 687 else 688 dst_target = 0; 689 dst_sg = sg_next(dst_sg); 690 } 691 } 692 693 /* Null descriptor with invalid toggle bit */ 694 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle)); 695 696 /* Ensure that descriptors have been written to memory */ 697 wmb(); 698 699 /* Flip toggle bit in header */ 700 flexrm_flip_header_toggle(orig_desc_ptr); 701 702 return desc_ptr; 703 } 704 705 static bool flexrm_sba_sanity_check(struct brcm_message *msg) 706 { 707 u32 i; 708 709 if (!msg->sba.cmds || !msg->sba.cmds_count) 710 return false; 711 712 for (i = 0; i < msg->sba.cmds_count; i++) { 713 if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) || 714 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) && 715 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)) 716 return false; 717 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) && 718 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK)) 719 return false; 720 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) && 721 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK)) 722 return false; 723 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) && 724 (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK)) 725 return false; 726 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) && 727 (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK)) 728 return false; 729 } 730 731 return true; 732 } 733 734 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg) 735 { 736 u32 i, cnt; 737 738 cnt = 0; 739 for (i = 0; i < msg->sba.cmds_count; i++) { 740 cnt++; 741 742 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) || 743 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) 744 cnt++; 745 746 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) 747 cnt++; 748 749 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) 750 cnt++; 751 } 752 753 return cnt; 754 } 755 756 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt, 757 u32 reqid, void *desc_ptr, u32 toggle, 758 void *start_desc, void *end_desc) 759 { 760 u64 d; 761 u32 i, nhpos = 0; 762 struct brcm_sba_command *c; 763 void *orig_desc_ptr = desc_ptr; 764 765 /* Convert SBA commands into descriptors */ 766 for (i = 0; i < msg->sba.cmds_count; i++) { 767 c = &msg->sba.cmds[i]; 768 769 if ((c->flags & BRCM_SBA_CMD_HAS_RESP) && 770 (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) { 771 /* Destination response descriptor */ 772 d = flexrm_dst_desc(c->resp, c->resp_len); 773 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 774 d, &desc_ptr, &toggle, 775 start_desc, end_desc); 776 nhpos++; 777 } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) { 778 /* Destination response with tlast descriptor */ 779 d = flexrm_dstt_desc(c->resp, c->resp_len); 780 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 781 d, &desc_ptr, &toggle, 782 start_desc, end_desc); 783 nhpos++; 784 } 785 786 if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) { 787 /* Destination with tlast descriptor */ 788 d = flexrm_dstt_desc(c->data, c->data_len); 789 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 790 d, &desc_ptr, &toggle, 791 start_desc, end_desc); 792 nhpos++; 793 } 794 795 if (c->flags & BRCM_SBA_CMD_TYPE_B) { 796 /* Command as immediate descriptor */ 797 d = flexrm_imm_desc(c->cmd); 798 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 799 d, &desc_ptr, &toggle, 800 start_desc, end_desc); 801 nhpos++; 802 } else { 803 /* Command as immediate descriptor with tlast */ 804 d = flexrm_immt_desc(c->cmd); 805 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 806 d, &desc_ptr, &toggle, 807 start_desc, end_desc); 808 nhpos++; 809 } 810 811 if ((c->flags & BRCM_SBA_CMD_TYPE_B) || 812 (c->flags & BRCM_SBA_CMD_TYPE_C)) { 813 /* Source with tlast descriptor */ 814 d = flexrm_srct_desc(c->data, c->data_len); 815 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 816 d, &desc_ptr, &toggle, 817 start_desc, end_desc); 818 nhpos++; 819 } 820 } 821 822 /* Null descriptor with invalid toggle bit */ 823 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle)); 824 825 /* Ensure that descriptors have been written to memory */ 826 wmb(); 827 828 /* Flip toggle bit in header */ 829 flexrm_flip_header_toggle(orig_desc_ptr); 830 831 return desc_ptr; 832 } 833 834 static bool flexrm_sanity_check(struct brcm_message *msg) 835 { 836 if (!msg) 837 return false; 838 839 switch (msg->type) { 840 case BRCM_MESSAGE_SPU: 841 return flexrm_spu_sanity_check(msg); 842 case BRCM_MESSAGE_SBA: 843 return flexrm_sba_sanity_check(msg); 844 default: 845 return false; 846 }; 847 } 848 849 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg) 850 { 851 if (!msg) 852 return 0; 853 854 switch (msg->type) { 855 case BRCM_MESSAGE_SPU: 856 return flexrm_spu_estimate_nonheader_desc_count(msg); 857 case BRCM_MESSAGE_SBA: 858 return flexrm_sba_estimate_nonheader_desc_count(msg); 859 default: 860 return 0; 861 }; 862 } 863 864 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg) 865 { 866 if (!dev || !msg) 867 return -EINVAL; 868 869 switch (msg->type) { 870 case BRCM_MESSAGE_SPU: 871 return flexrm_spu_dma_map(dev, msg); 872 default: 873 break; 874 } 875 876 return 0; 877 } 878 879 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg) 880 { 881 if (!dev || !msg) 882 return; 883 884 switch (msg->type) { 885 case BRCM_MESSAGE_SPU: 886 flexrm_spu_dma_unmap(dev, msg); 887 break; 888 default: 889 break; 890 } 891 } 892 893 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt, 894 u32 reqid, void *desc_ptr, u32 toggle, 895 void *start_desc, void *end_desc) 896 { 897 if (!msg || !desc_ptr || !start_desc || !end_desc) 898 return ERR_PTR(-ENOTSUPP); 899 900 if ((desc_ptr < start_desc) || (end_desc <= desc_ptr)) 901 return ERR_PTR(-ERANGE); 902 903 switch (msg->type) { 904 case BRCM_MESSAGE_SPU: 905 return flexrm_spu_write_descs(msg, nhcnt, reqid, 906 desc_ptr, toggle, 907 start_desc, end_desc); 908 case BRCM_MESSAGE_SBA: 909 return flexrm_sba_write_descs(msg, nhcnt, reqid, 910 desc_ptr, toggle, 911 start_desc, end_desc); 912 default: 913 return ERR_PTR(-ENOTSUPP); 914 }; 915 } 916 917 /* ====== FlexRM driver helper routines ===== */ 918 919 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox, 920 struct seq_file *file) 921 { 922 int i; 923 const char *state; 924 struct flexrm_ring *ring; 925 926 seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n", 927 "Ring#", "State", "BD_Addr", "BD_Size", 928 "Cmpl_Addr", "Cmpl_Size"); 929 930 for (i = 0; i < mbox->num_rings; i++) { 931 ring = &mbox->rings[i]; 932 if (readl(ring->regs + RING_CONTROL) & 933 BIT(CONTROL_ACTIVE_SHIFT)) 934 state = "active"; 935 else 936 state = "inactive"; 937 seq_printf(file, 938 "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n", 939 ring->num, state, 940 (unsigned long long)ring->bd_dma_base, 941 (u32)RING_BD_SIZE, 942 (unsigned long long)ring->cmpl_dma_base, 943 (u32)RING_CMPL_SIZE); 944 } 945 } 946 947 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox, 948 struct seq_file *file) 949 { 950 int i; 951 u32 val, bd_read_offset; 952 struct flexrm_ring *ring; 953 954 seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n", 955 "Ring#", "BD_Read", "BD_Write", 956 "Cmpl_Read", "Submitted", "Completed"); 957 958 for (i = 0; i < mbox->num_rings; i++) { 959 ring = &mbox->rings[i]; 960 bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR); 961 val = readl_relaxed(ring->regs + RING_BD_START_ADDR); 962 bd_read_offset *= RING_DESC_SIZE; 963 bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) - 964 ring->bd_dma_base); 965 seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n", 966 ring->num, 967 (u32)bd_read_offset, 968 (u32)ring->bd_write_offset, 969 (u32)ring->cmpl_read_offset, 970 (u32)atomic_read(&ring->msg_send_count), 971 (u32)atomic_read(&ring->msg_cmpl_count)); 972 } 973 } 974 975 static int flexrm_new_request(struct flexrm_ring *ring, 976 struct brcm_message *batch_msg, 977 struct brcm_message *msg) 978 { 979 void *next; 980 unsigned long flags; 981 u32 val, count, nhcnt; 982 u32 read_offset, write_offset; 983 bool exit_cleanup = false; 984 int ret = 0, reqid; 985 986 /* Do sanity check on message */ 987 if (!flexrm_sanity_check(msg)) 988 return -EIO; 989 msg->error = 0; 990 991 /* If no requests possible then save data pointer and goto done. */ 992 spin_lock_irqsave(&ring->lock, flags); 993 reqid = bitmap_find_free_region(ring->requests_bmap, 994 RING_MAX_REQ_COUNT, 0); 995 spin_unlock_irqrestore(&ring->lock, flags); 996 if (reqid < 0) 997 return -ENOSPC; 998 ring->requests[reqid] = msg; 999 1000 /* Do DMA mappings for the message */ 1001 ret = flexrm_dma_map(ring->mbox->dev, msg); 1002 if (ret < 0) { 1003 ring->requests[reqid] = NULL; 1004 spin_lock_irqsave(&ring->lock, flags); 1005 bitmap_release_region(ring->requests_bmap, reqid, 0); 1006 spin_unlock_irqrestore(&ring->lock, flags); 1007 return ret; 1008 } 1009 1010 /* Determine current HW BD read offset */ 1011 read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR); 1012 val = readl_relaxed(ring->regs + RING_BD_START_ADDR); 1013 read_offset *= RING_DESC_SIZE; 1014 read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base); 1015 1016 /* 1017 * Number required descriptors = number of non-header descriptors + 1018 * number of header descriptors + 1019 * 1x null descriptor 1020 */ 1021 nhcnt = flexrm_estimate_nonheader_desc_count(msg); 1022 count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1; 1023 1024 /* Check for available descriptor space. */ 1025 write_offset = ring->bd_write_offset; 1026 while (count) { 1027 if (!flexrm_is_next_table_desc(ring->bd_base + write_offset)) 1028 count--; 1029 write_offset += RING_DESC_SIZE; 1030 if (write_offset == RING_BD_SIZE) 1031 write_offset = 0x0; 1032 if (write_offset == read_offset) 1033 break; 1034 } 1035 if (count) { 1036 ret = -ENOSPC; 1037 exit_cleanup = true; 1038 goto exit; 1039 } 1040 1041 /* Write descriptors to ring */ 1042 next = flexrm_write_descs(msg, nhcnt, reqid, 1043 ring->bd_base + ring->bd_write_offset, 1044 RING_BD_TOGGLE_VALID(ring->bd_write_offset), 1045 ring->bd_base, ring->bd_base + RING_BD_SIZE); 1046 if (IS_ERR(next)) { 1047 ret = PTR_ERR(next); 1048 exit_cleanup = true; 1049 goto exit; 1050 } 1051 1052 /* Save ring BD write offset */ 1053 ring->bd_write_offset = (unsigned long)(next - ring->bd_base); 1054 1055 /* Increment number of messages sent */ 1056 atomic_inc_return(&ring->msg_send_count); 1057 1058 exit: 1059 /* Update error status in message */ 1060 msg->error = ret; 1061 1062 /* Cleanup if we failed */ 1063 if (exit_cleanup) { 1064 flexrm_dma_unmap(ring->mbox->dev, msg); 1065 ring->requests[reqid] = NULL; 1066 spin_lock_irqsave(&ring->lock, flags); 1067 bitmap_release_region(ring->requests_bmap, reqid, 0); 1068 spin_unlock_irqrestore(&ring->lock, flags); 1069 } 1070 1071 return ret; 1072 } 1073 1074 static int flexrm_process_completions(struct flexrm_ring *ring) 1075 { 1076 u64 desc; 1077 int err, count = 0; 1078 unsigned long flags; 1079 struct brcm_message *msg = NULL; 1080 u32 reqid, cmpl_read_offset, cmpl_write_offset; 1081 struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num]; 1082 1083 spin_lock_irqsave(&ring->lock, flags); 1084 1085 /* 1086 * Get current completion read and write offset 1087 * 1088 * Note: We should read completion write pointer at least once 1089 * after we get a MSI interrupt because HW maintains internal 1090 * MSI status which will allow next MSI interrupt only after 1091 * completion write pointer is read. 1092 */ 1093 cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR); 1094 cmpl_write_offset *= RING_DESC_SIZE; 1095 cmpl_read_offset = ring->cmpl_read_offset; 1096 ring->cmpl_read_offset = cmpl_write_offset; 1097 1098 spin_unlock_irqrestore(&ring->lock, flags); 1099 1100 /* For each completed request notify mailbox clients */ 1101 reqid = 0; 1102 while (cmpl_read_offset != cmpl_write_offset) { 1103 /* Dequeue next completion descriptor */ 1104 desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset)); 1105 1106 /* Next read offset */ 1107 cmpl_read_offset += RING_DESC_SIZE; 1108 if (cmpl_read_offset == RING_CMPL_SIZE) 1109 cmpl_read_offset = 0; 1110 1111 /* Decode error from completion descriptor */ 1112 err = flexrm_cmpl_desc_to_error(desc); 1113 if (err < 0) { 1114 dev_warn(ring->mbox->dev, 1115 "ring%d got completion desc=0x%lx with error %d\n", 1116 ring->num, (unsigned long)desc, err); 1117 } 1118 1119 /* Determine request id from completion descriptor */ 1120 reqid = flexrm_cmpl_desc_to_reqid(desc); 1121 1122 /* Determine message pointer based on reqid */ 1123 msg = ring->requests[reqid]; 1124 if (!msg) { 1125 dev_warn(ring->mbox->dev, 1126 "ring%d null msg pointer for completion desc=0x%lx\n", 1127 ring->num, (unsigned long)desc); 1128 continue; 1129 } 1130 1131 /* Release reqid for recycling */ 1132 ring->requests[reqid] = NULL; 1133 spin_lock_irqsave(&ring->lock, flags); 1134 bitmap_release_region(ring->requests_bmap, reqid, 0); 1135 spin_unlock_irqrestore(&ring->lock, flags); 1136 1137 /* Unmap DMA mappings */ 1138 flexrm_dma_unmap(ring->mbox->dev, msg); 1139 1140 /* Give-back message to mailbox client */ 1141 msg->error = err; 1142 mbox_chan_received_data(chan, msg); 1143 1144 /* Increment number of completions processed */ 1145 atomic_inc_return(&ring->msg_cmpl_count); 1146 count++; 1147 } 1148 1149 return count; 1150 } 1151 1152 /* ====== FlexRM Debugfs callbacks ====== */ 1153 1154 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset) 1155 { 1156 struct flexrm_mbox *mbox = dev_get_drvdata(file->private); 1157 1158 /* Write config in file */ 1159 flexrm_write_config_in_seqfile(mbox, file); 1160 1161 return 0; 1162 } 1163 1164 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset) 1165 { 1166 struct flexrm_mbox *mbox = dev_get_drvdata(file->private); 1167 1168 /* Write stats in file */ 1169 flexrm_write_stats_in_seqfile(mbox, file); 1170 1171 return 0; 1172 } 1173 1174 /* ====== FlexRM interrupt handler ===== */ 1175 1176 static irqreturn_t flexrm_irq_event(int irq, void *dev_id) 1177 { 1178 /* We only have MSI for completions so just wakeup IRQ thread */ 1179 /* Ring related errors will be informed via completion descriptors */ 1180 1181 return IRQ_WAKE_THREAD; 1182 } 1183 1184 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id) 1185 { 1186 flexrm_process_completions(dev_id); 1187 1188 return IRQ_HANDLED; 1189 } 1190 1191 /* ====== FlexRM mailbox callbacks ===== */ 1192 1193 static int flexrm_send_data(struct mbox_chan *chan, void *data) 1194 { 1195 int i, rc; 1196 struct flexrm_ring *ring = chan->con_priv; 1197 struct brcm_message *msg = data; 1198 1199 if (msg->type == BRCM_MESSAGE_BATCH) { 1200 for (i = msg->batch.msgs_queued; 1201 i < msg->batch.msgs_count; i++) { 1202 rc = flexrm_new_request(ring, msg, 1203 &msg->batch.msgs[i]); 1204 if (rc) { 1205 msg->error = rc; 1206 return rc; 1207 } 1208 msg->batch.msgs_queued++; 1209 } 1210 return 0; 1211 } 1212 1213 return flexrm_new_request(ring, NULL, data); 1214 } 1215 1216 static bool flexrm_peek_data(struct mbox_chan *chan) 1217 { 1218 int cnt = flexrm_process_completions(chan->con_priv); 1219 1220 return (cnt > 0) ? true : false; 1221 } 1222 1223 static int flexrm_startup(struct mbox_chan *chan) 1224 { 1225 u64 d; 1226 u32 val, off; 1227 int ret = 0; 1228 dma_addr_t next_addr; 1229 struct flexrm_ring *ring = chan->con_priv; 1230 1231 /* Allocate BD memory */ 1232 ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool, 1233 GFP_KERNEL, &ring->bd_dma_base); 1234 if (!ring->bd_base) { 1235 dev_err(ring->mbox->dev, 1236 "can't allocate BD memory for ring%d\n", 1237 ring->num); 1238 ret = -ENOMEM; 1239 goto fail; 1240 } 1241 1242 /* Configure next table pointer entries in BD memory */ 1243 for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) { 1244 next_addr = off + RING_DESC_SIZE; 1245 if (next_addr == RING_BD_SIZE) 1246 next_addr = 0; 1247 next_addr += ring->bd_dma_base; 1248 if (RING_BD_ALIGN_CHECK(next_addr)) 1249 d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off), 1250 next_addr); 1251 else 1252 d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off)); 1253 flexrm_write_desc(ring->bd_base + off, d); 1254 } 1255 1256 /* Allocate completion memory */ 1257 ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool, 1258 GFP_KERNEL, &ring->cmpl_dma_base); 1259 if (!ring->cmpl_base) { 1260 dev_err(ring->mbox->dev, 1261 "can't allocate completion memory for ring%d\n", 1262 ring->num); 1263 ret = -ENOMEM; 1264 goto fail_free_bd_memory; 1265 } 1266 1267 /* Request IRQ */ 1268 if (ring->irq == UINT_MAX) { 1269 dev_err(ring->mbox->dev, 1270 "ring%d IRQ not available\n", ring->num); 1271 ret = -ENODEV; 1272 goto fail_free_cmpl_memory; 1273 } 1274 ret = request_threaded_irq(ring->irq, 1275 flexrm_irq_event, 1276 flexrm_irq_thread, 1277 0, dev_name(ring->mbox->dev), ring); 1278 if (ret) { 1279 dev_err(ring->mbox->dev, 1280 "failed to request ring%d IRQ\n", ring->num); 1281 goto fail_free_cmpl_memory; 1282 } 1283 ring->irq_requested = true; 1284 1285 /* Set IRQ affinity hint */ 1286 ring->irq_aff_hint = CPU_MASK_NONE; 1287 val = ring->mbox->num_rings; 1288 val = (num_online_cpus() < val) ? val / num_online_cpus() : 1; 1289 cpumask_set_cpu((ring->num / val) % num_online_cpus(), 1290 &ring->irq_aff_hint); 1291 ret = irq_update_affinity_hint(ring->irq, &ring->irq_aff_hint); 1292 if (ret) { 1293 dev_err(ring->mbox->dev, 1294 "failed to set IRQ affinity hint for ring%d\n", 1295 ring->num); 1296 goto fail_free_irq; 1297 } 1298 1299 /* Disable/inactivate ring */ 1300 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1301 1302 /* Program BD start address */ 1303 val = BD_START_ADDR_VALUE(ring->bd_dma_base); 1304 writel_relaxed(val, ring->regs + RING_BD_START_ADDR); 1305 1306 /* BD write pointer will be same as HW write pointer */ 1307 ring->bd_write_offset = 1308 readl_relaxed(ring->regs + RING_BD_WRITE_PTR); 1309 ring->bd_write_offset *= RING_DESC_SIZE; 1310 1311 /* Program completion start address */ 1312 val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base); 1313 writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR); 1314 1315 /* Completion read pointer will be same as HW write pointer */ 1316 ring->cmpl_read_offset = 1317 readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR); 1318 ring->cmpl_read_offset *= RING_DESC_SIZE; 1319 1320 /* Read ring Tx, Rx, and Outstanding counts to clear */ 1321 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS); 1322 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS); 1323 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS); 1324 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS); 1325 readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND); 1326 1327 /* Configure RING_MSI_CONTROL */ 1328 val = 0; 1329 val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT); 1330 val |= BIT(MSI_ENABLE_SHIFT); 1331 val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT; 1332 writel_relaxed(val, ring->regs + RING_MSI_CONTROL); 1333 1334 /* Enable/activate ring */ 1335 val = BIT(CONTROL_ACTIVE_SHIFT); 1336 writel_relaxed(val, ring->regs + RING_CONTROL); 1337 1338 /* Reset stats to zero */ 1339 atomic_set(&ring->msg_send_count, 0); 1340 atomic_set(&ring->msg_cmpl_count, 0); 1341 1342 return 0; 1343 1344 fail_free_irq: 1345 free_irq(ring->irq, ring); 1346 ring->irq_requested = false; 1347 fail_free_cmpl_memory: 1348 dma_pool_free(ring->mbox->cmpl_pool, 1349 ring->cmpl_base, ring->cmpl_dma_base); 1350 ring->cmpl_base = NULL; 1351 fail_free_bd_memory: 1352 dma_pool_free(ring->mbox->bd_pool, 1353 ring->bd_base, ring->bd_dma_base); 1354 ring->bd_base = NULL; 1355 fail: 1356 return ret; 1357 } 1358 1359 static void flexrm_shutdown(struct mbox_chan *chan) 1360 { 1361 u32 reqid; 1362 unsigned int timeout; 1363 struct brcm_message *msg; 1364 struct flexrm_ring *ring = chan->con_priv; 1365 1366 /* Disable/inactivate ring */ 1367 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1368 1369 /* Set ring flush state */ 1370 timeout = 1000; /* timeout of 1s */ 1371 writel_relaxed(BIT(CONTROL_FLUSH_SHIFT), 1372 ring->regs + RING_CONTROL); 1373 do { 1374 if (readl_relaxed(ring->regs + RING_FLUSH_DONE) & 1375 FLUSH_DONE_MASK) 1376 break; 1377 mdelay(1); 1378 } while (--timeout); 1379 if (!timeout) 1380 dev_err(ring->mbox->dev, 1381 "setting ring%d flush state timedout\n", ring->num); 1382 1383 /* Clear ring flush state */ 1384 timeout = 1000; /* timeout of 1s */ 1385 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1386 do { 1387 if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) & 1388 FLUSH_DONE_MASK)) 1389 break; 1390 mdelay(1); 1391 } while (--timeout); 1392 if (!timeout) 1393 dev_err(ring->mbox->dev, 1394 "clearing ring%d flush state timedout\n", ring->num); 1395 1396 /* Abort all in-flight requests */ 1397 for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) { 1398 msg = ring->requests[reqid]; 1399 if (!msg) 1400 continue; 1401 1402 /* Release reqid for recycling */ 1403 ring->requests[reqid] = NULL; 1404 1405 /* Unmap DMA mappings */ 1406 flexrm_dma_unmap(ring->mbox->dev, msg); 1407 1408 /* Give-back message to mailbox client */ 1409 msg->error = -EIO; 1410 mbox_chan_received_data(chan, msg); 1411 } 1412 1413 /* Clear requests bitmap */ 1414 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT); 1415 1416 /* Release IRQ */ 1417 if (ring->irq_requested) { 1418 irq_update_affinity_hint(ring->irq, NULL); 1419 free_irq(ring->irq, ring); 1420 ring->irq_requested = false; 1421 } 1422 1423 /* Free-up completion descriptor ring */ 1424 if (ring->cmpl_base) { 1425 dma_pool_free(ring->mbox->cmpl_pool, 1426 ring->cmpl_base, ring->cmpl_dma_base); 1427 ring->cmpl_base = NULL; 1428 } 1429 1430 /* Free-up BD descriptor ring */ 1431 if (ring->bd_base) { 1432 dma_pool_free(ring->mbox->bd_pool, 1433 ring->bd_base, ring->bd_dma_base); 1434 ring->bd_base = NULL; 1435 } 1436 } 1437 1438 static const struct mbox_chan_ops flexrm_mbox_chan_ops = { 1439 .send_data = flexrm_send_data, 1440 .startup = flexrm_startup, 1441 .shutdown = flexrm_shutdown, 1442 .peek_data = flexrm_peek_data, 1443 }; 1444 1445 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr, 1446 const struct of_phandle_args *pa) 1447 { 1448 struct mbox_chan *chan; 1449 struct flexrm_ring *ring; 1450 1451 if (pa->args_count < 3) 1452 return ERR_PTR(-EINVAL); 1453 1454 if (pa->args[0] >= cntlr->num_chans) 1455 return ERR_PTR(-ENOENT); 1456 1457 if (pa->args[1] > MSI_COUNT_MASK) 1458 return ERR_PTR(-EINVAL); 1459 1460 if (pa->args[2] > MSI_TIMER_VAL_MASK) 1461 return ERR_PTR(-EINVAL); 1462 1463 chan = &cntlr->chans[pa->args[0]]; 1464 ring = chan->con_priv; 1465 ring->msi_count_threshold = pa->args[1]; 1466 ring->msi_timer_val = pa->args[2]; 1467 1468 return chan; 1469 } 1470 1471 /* ====== FlexRM platform driver ===== */ 1472 1473 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg) 1474 { 1475 struct device *dev = msi_desc_to_dev(desc); 1476 struct flexrm_mbox *mbox = dev_get_drvdata(dev); 1477 struct flexrm_ring *ring = &mbox->rings[desc->msi_index]; 1478 1479 /* Configure per-Ring MSI registers */ 1480 writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS); 1481 writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS); 1482 writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE); 1483 } 1484 1485 static int flexrm_mbox_probe(struct platform_device *pdev) 1486 { 1487 int index, ret = 0; 1488 void __iomem *regs; 1489 void __iomem *regs_end; 1490 struct resource *iomem; 1491 struct flexrm_ring *ring; 1492 struct flexrm_mbox *mbox; 1493 struct device *dev = &pdev->dev; 1494 1495 /* Allocate driver mailbox struct */ 1496 mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); 1497 if (!mbox) { 1498 ret = -ENOMEM; 1499 goto fail; 1500 } 1501 mbox->dev = dev; 1502 platform_set_drvdata(pdev, mbox); 1503 1504 /* Get resource for registers */ 1505 iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1506 if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) { 1507 ret = -ENODEV; 1508 goto fail; 1509 } 1510 1511 /* Map registers of all rings */ 1512 mbox->regs = devm_ioremap_resource(&pdev->dev, iomem); 1513 if (IS_ERR(mbox->regs)) { 1514 ret = PTR_ERR(mbox->regs); 1515 goto fail; 1516 } 1517 regs_end = mbox->regs + resource_size(iomem); 1518 1519 /* Scan and count available rings */ 1520 mbox->num_rings = 0; 1521 for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) { 1522 if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC) 1523 mbox->num_rings++; 1524 } 1525 if (!mbox->num_rings) { 1526 ret = -ENODEV; 1527 goto fail; 1528 } 1529 1530 /* Allocate driver ring structs */ 1531 ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL); 1532 if (!ring) { 1533 ret = -ENOMEM; 1534 goto fail; 1535 } 1536 mbox->rings = ring; 1537 1538 /* Initialize members of driver ring structs */ 1539 regs = mbox->regs; 1540 for (index = 0; index < mbox->num_rings; index++) { 1541 ring = &mbox->rings[index]; 1542 ring->num = index; 1543 ring->mbox = mbox; 1544 while ((regs < regs_end) && 1545 (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC)) 1546 regs += RING_REGS_SIZE; 1547 if (regs_end <= regs) { 1548 ret = -ENODEV; 1549 goto fail; 1550 } 1551 ring->regs = regs; 1552 regs += RING_REGS_SIZE; 1553 ring->irq = UINT_MAX; 1554 ring->irq_requested = false; 1555 ring->msi_timer_val = MSI_TIMER_VAL_MASK; 1556 ring->msi_count_threshold = 0x1; 1557 memset(ring->requests, 0, sizeof(ring->requests)); 1558 ring->bd_base = NULL; 1559 ring->bd_dma_base = 0; 1560 ring->cmpl_base = NULL; 1561 ring->cmpl_dma_base = 0; 1562 atomic_set(&ring->msg_send_count, 0); 1563 atomic_set(&ring->msg_cmpl_count, 0); 1564 spin_lock_init(&ring->lock); 1565 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT); 1566 ring->cmpl_read_offset = 0; 1567 } 1568 1569 /* FlexRM is capable of 40-bit physical addresses only */ 1570 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); 1571 if (ret) { 1572 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1573 if (ret) 1574 goto fail; 1575 } 1576 1577 /* Create DMA pool for ring BD memory */ 1578 mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE, 1579 1 << RING_BD_ALIGN_ORDER, 0); 1580 if (!mbox->bd_pool) { 1581 ret = -ENOMEM; 1582 goto fail; 1583 } 1584 1585 /* Create DMA pool for ring completion memory */ 1586 mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE, 1587 1 << RING_CMPL_ALIGN_ORDER, 0); 1588 if (!mbox->cmpl_pool) { 1589 ret = -ENOMEM; 1590 goto fail_destroy_bd_pool; 1591 } 1592 1593 /* Allocate platform MSIs for each ring */ 1594 ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings, 1595 flexrm_mbox_msi_write); 1596 if (ret) 1597 goto fail_destroy_cmpl_pool; 1598 1599 /* Save alloced IRQ numbers for each ring */ 1600 for (index = 0; index < mbox->num_rings; index++) 1601 mbox->rings[index].irq = msi_get_virq(dev, index); 1602 1603 /* Check availability of debugfs */ 1604 if (!debugfs_initialized()) 1605 goto skip_debugfs; 1606 1607 /* Create debugfs root entry */ 1608 mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL); 1609 1610 /* Create debugfs config entry */ 1611 debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root, 1612 flexrm_debugfs_conf_show); 1613 1614 /* Create debugfs stats entry */ 1615 debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root, 1616 flexrm_debugfs_stats_show); 1617 1618 skip_debugfs: 1619 1620 /* Initialize mailbox controller */ 1621 mbox->controller.txdone_irq = false; 1622 mbox->controller.txdone_poll = false; 1623 mbox->controller.ops = &flexrm_mbox_chan_ops; 1624 mbox->controller.dev = dev; 1625 mbox->controller.num_chans = mbox->num_rings; 1626 mbox->controller.of_xlate = flexrm_mbox_of_xlate; 1627 mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings, 1628 sizeof(*mbox->controller.chans), GFP_KERNEL); 1629 if (!mbox->controller.chans) { 1630 ret = -ENOMEM; 1631 goto fail_free_debugfs_root; 1632 } 1633 for (index = 0; index < mbox->num_rings; index++) 1634 mbox->controller.chans[index].con_priv = &mbox->rings[index]; 1635 1636 /* Register mailbox controller */ 1637 ret = devm_mbox_controller_register(dev, &mbox->controller); 1638 if (ret) 1639 goto fail_free_debugfs_root; 1640 1641 dev_info(dev, "registered flexrm mailbox with %d channels\n", 1642 mbox->controller.num_chans); 1643 1644 return 0; 1645 1646 fail_free_debugfs_root: 1647 debugfs_remove_recursive(mbox->root); 1648 platform_msi_domain_free_irqs(dev); 1649 fail_destroy_cmpl_pool: 1650 dma_pool_destroy(mbox->cmpl_pool); 1651 fail_destroy_bd_pool: 1652 dma_pool_destroy(mbox->bd_pool); 1653 fail: 1654 return ret; 1655 } 1656 1657 static int flexrm_mbox_remove(struct platform_device *pdev) 1658 { 1659 struct device *dev = &pdev->dev; 1660 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1661 1662 debugfs_remove_recursive(mbox->root); 1663 1664 platform_msi_domain_free_irqs(dev); 1665 1666 dma_pool_destroy(mbox->cmpl_pool); 1667 dma_pool_destroy(mbox->bd_pool); 1668 1669 return 0; 1670 } 1671 1672 static const struct of_device_id flexrm_mbox_of_match[] = { 1673 { .compatible = "brcm,iproc-flexrm-mbox", }, 1674 {}, 1675 }; 1676 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match); 1677 1678 static struct platform_driver flexrm_mbox_driver = { 1679 .driver = { 1680 .name = "brcm-flexrm-mbox", 1681 .of_match_table = flexrm_mbox_of_match, 1682 }, 1683 .probe = flexrm_mbox_probe, 1684 .remove = flexrm_mbox_remove, 1685 }; 1686 module_platform_driver(flexrm_mbox_driver); 1687 1688 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1689 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver"); 1690 MODULE_LICENSE("GPL v2"); 1691