1 /* 2 * Copyright (C) 2017 Broadcom 3 * 4 * This program is free software; you can redistribute it and/or 5 * modify it under the terms of the GNU General Public License as 6 * published by the Free Software Foundation version 2. 7 * 8 * This program is distributed "as is" WITHOUT ANY WARRANTY of any 9 * kind, whether express or implied; without even the implied warranty 10 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 */ 13 14 /* 15 * Broadcom FlexRM Mailbox Driver 16 * 17 * Each Broadcom FlexSparx4 offload engine is implemented as an 18 * extension to Broadcom FlexRM ring manager. The FlexRM ring 19 * manager provides a set of rings which can be used to submit 20 * work to a FlexSparx4 offload engine. 21 * 22 * This driver creates a mailbox controller using a set of FlexRM 23 * rings where each mailbox channel represents a separate FlexRM ring. 24 */ 25 26 #include <asm/barrier.h> 27 #include <asm/byteorder.h> 28 #include <linux/atomic.h> 29 #include <linux/bitmap.h> 30 #include <linux/debugfs.h> 31 #include <linux/delay.h> 32 #include <linux/device.h> 33 #include <linux/dma-mapping.h> 34 #include <linux/dmapool.h> 35 #include <linux/err.h> 36 #include <linux/interrupt.h> 37 #include <linux/kernel.h> 38 #include <linux/mailbox_controller.h> 39 #include <linux/mailbox_client.h> 40 #include <linux/mailbox/brcm-message.h> 41 #include <linux/module.h> 42 #include <linux/msi.h> 43 #include <linux/of_address.h> 44 #include <linux/of_irq.h> 45 #include <linux/platform_device.h> 46 #include <linux/spinlock.h> 47 48 /* ====== FlexRM register defines ===== */ 49 50 /* FlexRM configuration */ 51 #define RING_REGS_SIZE 0x10000 52 #define RING_DESC_SIZE 8 53 #define RING_DESC_INDEX(offset) \ 54 ((offset) / RING_DESC_SIZE) 55 #define RING_DESC_OFFSET(index) \ 56 ((index) * RING_DESC_SIZE) 57 #define RING_MAX_REQ_COUNT 1024 58 #define RING_BD_ALIGN_ORDER 12 59 #define RING_BD_ALIGN_CHECK(addr) \ 60 (!((addr) & ((0x1 << RING_BD_ALIGN_ORDER) - 1))) 61 #define RING_BD_TOGGLE_INVALID(offset) \ 62 (((offset) >> RING_BD_ALIGN_ORDER) & 0x1) 63 #define RING_BD_TOGGLE_VALID(offset) \ 64 (!RING_BD_TOGGLE_INVALID(offset)) 65 #define RING_BD_DESC_PER_REQ 32 66 #define RING_BD_DESC_COUNT \ 67 (RING_MAX_REQ_COUNT * RING_BD_DESC_PER_REQ) 68 #define RING_BD_SIZE \ 69 (RING_BD_DESC_COUNT * RING_DESC_SIZE) 70 #define RING_CMPL_ALIGN_ORDER 13 71 #define RING_CMPL_DESC_COUNT RING_MAX_REQ_COUNT 72 #define RING_CMPL_SIZE \ 73 (RING_CMPL_DESC_COUNT * RING_DESC_SIZE) 74 #define RING_VER_MAGIC 0x76303031 75 76 /* Per-Ring register offsets */ 77 #define RING_VER 0x000 78 #define RING_BD_START_ADDR 0x004 79 #define RING_BD_READ_PTR 0x008 80 #define RING_BD_WRITE_PTR 0x00c 81 #define RING_BD_READ_PTR_DDR_LS 0x010 82 #define RING_BD_READ_PTR_DDR_MS 0x014 83 #define RING_CMPL_START_ADDR 0x018 84 #define RING_CMPL_WRITE_PTR 0x01c 85 #define RING_NUM_REQ_RECV_LS 0x020 86 #define RING_NUM_REQ_RECV_MS 0x024 87 #define RING_NUM_REQ_TRANS_LS 0x028 88 #define RING_NUM_REQ_TRANS_MS 0x02c 89 #define RING_NUM_REQ_OUTSTAND 0x030 90 #define RING_CONTROL 0x034 91 #define RING_FLUSH_DONE 0x038 92 #define RING_MSI_ADDR_LS 0x03c 93 #define RING_MSI_ADDR_MS 0x040 94 #define RING_MSI_CONTROL 0x048 95 #define RING_BD_READ_PTR_DDR_CONTROL 0x04c 96 #define RING_MSI_DATA_VALUE 0x064 97 98 /* Register RING_BD_START_ADDR fields */ 99 #define BD_LAST_UPDATE_HW_SHIFT 28 100 #define BD_LAST_UPDATE_HW_MASK 0x1 101 #define BD_START_ADDR_VALUE(pa) \ 102 ((u32)((((dma_addr_t)(pa)) >> RING_BD_ALIGN_ORDER) & 0x0fffffff)) 103 #define BD_START_ADDR_DECODE(val) \ 104 ((dma_addr_t)((val) & 0x0fffffff) << RING_BD_ALIGN_ORDER) 105 106 /* Register RING_CMPL_START_ADDR fields */ 107 #define CMPL_START_ADDR_VALUE(pa) \ 108 ((u32)((((u64)(pa)) >> RING_CMPL_ALIGN_ORDER) & 0x07ffffff)) 109 110 /* Register RING_CONTROL fields */ 111 #define CONTROL_MASK_DISABLE_CONTROL 12 112 #define CONTROL_FLUSH_SHIFT 5 113 #define CONTROL_ACTIVE_SHIFT 4 114 #define CONTROL_RATE_ADAPT_MASK 0xf 115 #define CONTROL_RATE_DYNAMIC 0x0 116 #define CONTROL_RATE_FAST 0x8 117 #define CONTROL_RATE_MEDIUM 0x9 118 #define CONTROL_RATE_SLOW 0xa 119 #define CONTROL_RATE_IDLE 0xb 120 121 /* Register RING_FLUSH_DONE fields */ 122 #define FLUSH_DONE_MASK 0x1 123 124 /* Register RING_MSI_CONTROL fields */ 125 #define MSI_TIMER_VAL_SHIFT 16 126 #define MSI_TIMER_VAL_MASK 0xffff 127 #define MSI_ENABLE_SHIFT 15 128 #define MSI_ENABLE_MASK 0x1 129 #define MSI_COUNT_SHIFT 0 130 #define MSI_COUNT_MASK 0x3ff 131 132 /* Register RING_BD_READ_PTR_DDR_CONTROL fields */ 133 #define BD_READ_PTR_DDR_TIMER_VAL_SHIFT 16 134 #define BD_READ_PTR_DDR_TIMER_VAL_MASK 0xffff 135 #define BD_READ_PTR_DDR_ENABLE_SHIFT 15 136 #define BD_READ_PTR_DDR_ENABLE_MASK 0x1 137 138 /* ====== FlexRM ring descriptor defines ===== */ 139 140 /* Completion descriptor format */ 141 #define CMPL_OPAQUE_SHIFT 0 142 #define CMPL_OPAQUE_MASK 0xffff 143 #define CMPL_ENGINE_STATUS_SHIFT 16 144 #define CMPL_ENGINE_STATUS_MASK 0xffff 145 #define CMPL_DME_STATUS_SHIFT 32 146 #define CMPL_DME_STATUS_MASK 0xffff 147 #define CMPL_RM_STATUS_SHIFT 48 148 #define CMPL_RM_STATUS_MASK 0xffff 149 150 /* Completion DME status code */ 151 #define DME_STATUS_MEM_COR_ERR BIT(0) 152 #define DME_STATUS_MEM_UCOR_ERR BIT(1) 153 #define DME_STATUS_FIFO_UNDERFLOW BIT(2) 154 #define DME_STATUS_FIFO_OVERFLOW BIT(3) 155 #define DME_STATUS_RRESP_ERR BIT(4) 156 #define DME_STATUS_BRESP_ERR BIT(5) 157 #define DME_STATUS_ERROR_MASK (DME_STATUS_MEM_COR_ERR | \ 158 DME_STATUS_MEM_UCOR_ERR | \ 159 DME_STATUS_FIFO_UNDERFLOW | \ 160 DME_STATUS_FIFO_OVERFLOW | \ 161 DME_STATUS_RRESP_ERR | \ 162 DME_STATUS_BRESP_ERR) 163 164 /* Completion RM status code */ 165 #define RM_STATUS_CODE_SHIFT 0 166 #define RM_STATUS_CODE_MASK 0x3ff 167 #define RM_STATUS_CODE_GOOD 0x0 168 #define RM_STATUS_CODE_AE_TIMEOUT 0x3ff 169 170 /* General descriptor format */ 171 #define DESC_TYPE_SHIFT 60 172 #define DESC_TYPE_MASK 0xf 173 #define DESC_PAYLOAD_SHIFT 0 174 #define DESC_PAYLOAD_MASK 0x0fffffffffffffff 175 176 /* Null descriptor format */ 177 #define NULL_TYPE 0 178 #define NULL_TOGGLE_SHIFT 58 179 #define NULL_TOGGLE_MASK 0x1 180 181 /* Header descriptor format */ 182 #define HEADER_TYPE 1 183 #define HEADER_TOGGLE_SHIFT 58 184 #define HEADER_TOGGLE_MASK 0x1 185 #define HEADER_ENDPKT_SHIFT 57 186 #define HEADER_ENDPKT_MASK 0x1 187 #define HEADER_STARTPKT_SHIFT 56 188 #define HEADER_STARTPKT_MASK 0x1 189 #define HEADER_BDCOUNT_SHIFT 36 190 #define HEADER_BDCOUNT_MASK 0x1f 191 #define HEADER_BDCOUNT_MAX HEADER_BDCOUNT_MASK 192 #define HEADER_FLAGS_SHIFT 16 193 #define HEADER_FLAGS_MASK 0xffff 194 #define HEADER_OPAQUE_SHIFT 0 195 #define HEADER_OPAQUE_MASK 0xffff 196 197 /* Source (SRC) descriptor format */ 198 #define SRC_TYPE 2 199 #define SRC_LENGTH_SHIFT 44 200 #define SRC_LENGTH_MASK 0xffff 201 #define SRC_ADDR_SHIFT 0 202 #define SRC_ADDR_MASK 0x00000fffffffffff 203 204 /* Destination (DST) descriptor format */ 205 #define DST_TYPE 3 206 #define DST_LENGTH_SHIFT 44 207 #define DST_LENGTH_MASK 0xffff 208 #define DST_ADDR_SHIFT 0 209 #define DST_ADDR_MASK 0x00000fffffffffff 210 211 /* Immediate (IMM) descriptor format */ 212 #define IMM_TYPE 4 213 #define IMM_DATA_SHIFT 0 214 #define IMM_DATA_MASK 0x0fffffffffffffff 215 216 /* Next pointer (NPTR) descriptor format */ 217 #define NPTR_TYPE 5 218 #define NPTR_TOGGLE_SHIFT 58 219 #define NPTR_TOGGLE_MASK 0x1 220 #define NPTR_ADDR_SHIFT 0 221 #define NPTR_ADDR_MASK 0x00000fffffffffff 222 223 /* Mega source (MSRC) descriptor format */ 224 #define MSRC_TYPE 6 225 #define MSRC_LENGTH_SHIFT 44 226 #define MSRC_LENGTH_MASK 0xffff 227 #define MSRC_ADDR_SHIFT 0 228 #define MSRC_ADDR_MASK 0x00000fffffffffff 229 230 /* Mega destination (MDST) descriptor format */ 231 #define MDST_TYPE 7 232 #define MDST_LENGTH_SHIFT 44 233 #define MDST_LENGTH_MASK 0xffff 234 #define MDST_ADDR_SHIFT 0 235 #define MDST_ADDR_MASK 0x00000fffffffffff 236 237 /* Source with tlast (SRCT) descriptor format */ 238 #define SRCT_TYPE 8 239 #define SRCT_LENGTH_SHIFT 44 240 #define SRCT_LENGTH_MASK 0xffff 241 #define SRCT_ADDR_SHIFT 0 242 #define SRCT_ADDR_MASK 0x00000fffffffffff 243 244 /* Destination with tlast (DSTT) descriptor format */ 245 #define DSTT_TYPE 9 246 #define DSTT_LENGTH_SHIFT 44 247 #define DSTT_LENGTH_MASK 0xffff 248 #define DSTT_ADDR_SHIFT 0 249 #define DSTT_ADDR_MASK 0x00000fffffffffff 250 251 /* Immediate with tlast (IMMT) descriptor format */ 252 #define IMMT_TYPE 10 253 #define IMMT_DATA_SHIFT 0 254 #define IMMT_DATA_MASK 0x0fffffffffffffff 255 256 /* Descriptor helper macros */ 257 #define DESC_DEC(_d, _s, _m) (((_d) >> (_s)) & (_m)) 258 #define DESC_ENC(_d, _v, _s, _m) \ 259 do { \ 260 (_d) &= ~((u64)(_m) << (_s)); \ 261 (_d) |= (((u64)(_v) & (_m)) << (_s)); \ 262 } while (0) 263 264 /* ====== FlexRM data structures ===== */ 265 266 struct flexrm_ring { 267 /* Unprotected members */ 268 int num; 269 struct flexrm_mbox *mbox; 270 void __iomem *regs; 271 bool irq_requested; 272 unsigned int irq; 273 cpumask_t irq_aff_hint; 274 unsigned int msi_timer_val; 275 unsigned int msi_count_threshold; 276 struct brcm_message *requests[RING_MAX_REQ_COUNT]; 277 void *bd_base; 278 dma_addr_t bd_dma_base; 279 u32 bd_write_offset; 280 void *cmpl_base; 281 dma_addr_t cmpl_dma_base; 282 /* Atomic stats */ 283 atomic_t msg_send_count; 284 atomic_t msg_cmpl_count; 285 /* Protected members */ 286 spinlock_t lock; 287 DECLARE_BITMAP(requests_bmap, RING_MAX_REQ_COUNT); 288 u32 cmpl_read_offset; 289 }; 290 291 struct flexrm_mbox { 292 struct device *dev; 293 void __iomem *regs; 294 u32 num_rings; 295 struct flexrm_ring *rings; 296 struct dma_pool *bd_pool; 297 struct dma_pool *cmpl_pool; 298 struct dentry *root; 299 struct mbox_controller controller; 300 }; 301 302 /* ====== FlexRM ring descriptor helper routines ===== */ 303 304 static u64 flexrm_read_desc(void *desc_ptr) 305 { 306 return le64_to_cpu(*((u64 *)desc_ptr)); 307 } 308 309 static void flexrm_write_desc(void *desc_ptr, u64 desc) 310 { 311 *((u64 *)desc_ptr) = cpu_to_le64(desc); 312 } 313 314 static u32 flexrm_cmpl_desc_to_reqid(u64 cmpl_desc) 315 { 316 return (u32)(cmpl_desc & CMPL_OPAQUE_MASK); 317 } 318 319 static int flexrm_cmpl_desc_to_error(u64 cmpl_desc) 320 { 321 u32 status; 322 323 status = DESC_DEC(cmpl_desc, CMPL_DME_STATUS_SHIFT, 324 CMPL_DME_STATUS_MASK); 325 if (status & DME_STATUS_ERROR_MASK) 326 return -EIO; 327 328 status = DESC_DEC(cmpl_desc, CMPL_RM_STATUS_SHIFT, 329 CMPL_RM_STATUS_MASK); 330 status &= RM_STATUS_CODE_MASK; 331 if (status == RM_STATUS_CODE_AE_TIMEOUT) 332 return -ETIMEDOUT; 333 334 return 0; 335 } 336 337 static bool flexrm_is_next_table_desc(void *desc_ptr) 338 { 339 u64 desc = flexrm_read_desc(desc_ptr); 340 u32 type = DESC_DEC(desc, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 341 342 return (type == NPTR_TYPE) ? true : false; 343 } 344 345 static u64 flexrm_next_table_desc(u32 toggle, dma_addr_t next_addr) 346 { 347 u64 desc = 0; 348 349 DESC_ENC(desc, NPTR_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 350 DESC_ENC(desc, toggle, NPTR_TOGGLE_SHIFT, NPTR_TOGGLE_MASK); 351 DESC_ENC(desc, next_addr, NPTR_ADDR_SHIFT, NPTR_ADDR_MASK); 352 353 return desc; 354 } 355 356 static u64 flexrm_null_desc(u32 toggle) 357 { 358 u64 desc = 0; 359 360 DESC_ENC(desc, NULL_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 361 DESC_ENC(desc, toggle, NULL_TOGGLE_SHIFT, NULL_TOGGLE_MASK); 362 363 return desc; 364 } 365 366 static u32 flexrm_estimate_header_desc_count(u32 nhcnt) 367 { 368 u32 hcnt = nhcnt / HEADER_BDCOUNT_MAX; 369 370 if (!(nhcnt % HEADER_BDCOUNT_MAX)) 371 hcnt += 1; 372 373 return hcnt; 374 } 375 376 static void flexrm_flip_header_toggle(void *desc_ptr) 377 { 378 u64 desc = flexrm_read_desc(desc_ptr); 379 380 if (desc & ((u64)0x1 << HEADER_TOGGLE_SHIFT)) 381 desc &= ~((u64)0x1 << HEADER_TOGGLE_SHIFT); 382 else 383 desc |= ((u64)0x1 << HEADER_TOGGLE_SHIFT); 384 385 flexrm_write_desc(desc_ptr, desc); 386 } 387 388 static u64 flexrm_header_desc(u32 toggle, u32 startpkt, u32 endpkt, 389 u32 bdcount, u32 flags, u32 opaque) 390 { 391 u64 desc = 0; 392 393 DESC_ENC(desc, HEADER_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 394 DESC_ENC(desc, toggle, HEADER_TOGGLE_SHIFT, HEADER_TOGGLE_MASK); 395 DESC_ENC(desc, startpkt, HEADER_STARTPKT_SHIFT, HEADER_STARTPKT_MASK); 396 DESC_ENC(desc, endpkt, HEADER_ENDPKT_SHIFT, HEADER_ENDPKT_MASK); 397 DESC_ENC(desc, bdcount, HEADER_BDCOUNT_SHIFT, HEADER_BDCOUNT_MASK); 398 DESC_ENC(desc, flags, HEADER_FLAGS_SHIFT, HEADER_FLAGS_MASK); 399 DESC_ENC(desc, opaque, HEADER_OPAQUE_SHIFT, HEADER_OPAQUE_MASK); 400 401 return desc; 402 } 403 404 static void flexrm_enqueue_desc(u32 nhpos, u32 nhcnt, u32 reqid, 405 u64 desc, void **desc_ptr, u32 *toggle, 406 void *start_desc, void *end_desc) 407 { 408 u64 d; 409 u32 nhavail, _toggle, _startpkt, _endpkt, _bdcount; 410 411 /* Sanity check */ 412 if (nhcnt <= nhpos) 413 return; 414 415 /* 416 * Each request or packet start with a HEADER descriptor followed 417 * by one or more non-HEADER descriptors (SRC, SRCT, MSRC, DST, 418 * DSTT, MDST, IMM, and IMMT). The number of non-HEADER descriptors 419 * following a HEADER descriptor is represented by BDCOUNT field 420 * of HEADER descriptor. The max value of BDCOUNT field is 31 which 421 * means we can only have 31 non-HEADER descriptors following one 422 * HEADER descriptor. 423 * 424 * In general use, number of non-HEADER descriptors can easily go 425 * beyond 31. To tackle this situation, we have packet (or request) 426 * extenstion bits (STARTPKT and ENDPKT) in the HEADER descriptor. 427 * 428 * To use packet extension, the first HEADER descriptor of request 429 * (or packet) will have STARTPKT=1 and ENDPKT=0. The intermediate 430 * HEADER descriptors will have STARTPKT=0 and ENDPKT=0. The last 431 * HEADER descriptor will have STARTPKT=0 and ENDPKT=1. Also, the 432 * TOGGLE bit of the first HEADER will be set to invalid state to 433 * ensure that FlexRM does not start fetching descriptors till all 434 * descriptors are enqueued. The user of this function will flip 435 * the TOGGLE bit of first HEADER after all descriptors are 436 * enqueued. 437 */ 438 439 if ((nhpos % HEADER_BDCOUNT_MAX == 0) && (nhcnt - nhpos)) { 440 /* Prepare the header descriptor */ 441 nhavail = (nhcnt - nhpos); 442 _toggle = (nhpos == 0) ? !(*toggle) : (*toggle); 443 _startpkt = (nhpos == 0) ? 0x1 : 0x0; 444 _endpkt = (nhavail <= HEADER_BDCOUNT_MAX) ? 0x1 : 0x0; 445 _bdcount = (nhavail <= HEADER_BDCOUNT_MAX) ? 446 nhavail : HEADER_BDCOUNT_MAX; 447 if (nhavail <= HEADER_BDCOUNT_MAX) 448 _bdcount = nhavail; 449 else 450 _bdcount = HEADER_BDCOUNT_MAX; 451 d = flexrm_header_desc(_toggle, _startpkt, _endpkt, 452 _bdcount, 0x0, reqid); 453 454 /* Write header descriptor */ 455 flexrm_write_desc(*desc_ptr, d); 456 457 /* Point to next descriptor */ 458 *desc_ptr += sizeof(desc); 459 if (*desc_ptr == end_desc) 460 *desc_ptr = start_desc; 461 462 /* Skip next pointer descriptors */ 463 while (flexrm_is_next_table_desc(*desc_ptr)) { 464 *toggle = (*toggle) ? 0 : 1; 465 *desc_ptr += sizeof(desc); 466 if (*desc_ptr == end_desc) 467 *desc_ptr = start_desc; 468 } 469 } 470 471 /* Write desired descriptor */ 472 flexrm_write_desc(*desc_ptr, desc); 473 474 /* Point to next descriptor */ 475 *desc_ptr += sizeof(desc); 476 if (*desc_ptr == end_desc) 477 *desc_ptr = start_desc; 478 479 /* Skip next pointer descriptors */ 480 while (flexrm_is_next_table_desc(*desc_ptr)) { 481 *toggle = (*toggle) ? 0 : 1; 482 *desc_ptr += sizeof(desc); 483 if (*desc_ptr == end_desc) 484 *desc_ptr = start_desc; 485 } 486 } 487 488 static u64 flexrm_src_desc(dma_addr_t addr, unsigned int length) 489 { 490 u64 desc = 0; 491 492 DESC_ENC(desc, SRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 493 DESC_ENC(desc, length, SRC_LENGTH_SHIFT, SRC_LENGTH_MASK); 494 DESC_ENC(desc, addr, SRC_ADDR_SHIFT, SRC_ADDR_MASK); 495 496 return desc; 497 } 498 499 static u64 flexrm_msrc_desc(dma_addr_t addr, unsigned int length_div_16) 500 { 501 u64 desc = 0; 502 503 DESC_ENC(desc, MSRC_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 504 DESC_ENC(desc, length_div_16, MSRC_LENGTH_SHIFT, MSRC_LENGTH_MASK); 505 DESC_ENC(desc, addr, MSRC_ADDR_SHIFT, MSRC_ADDR_MASK); 506 507 return desc; 508 } 509 510 static u64 flexrm_dst_desc(dma_addr_t addr, unsigned int length) 511 { 512 u64 desc = 0; 513 514 DESC_ENC(desc, DST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 515 DESC_ENC(desc, length, DST_LENGTH_SHIFT, DST_LENGTH_MASK); 516 DESC_ENC(desc, addr, DST_ADDR_SHIFT, DST_ADDR_MASK); 517 518 return desc; 519 } 520 521 static u64 flexrm_mdst_desc(dma_addr_t addr, unsigned int length_div_16) 522 { 523 u64 desc = 0; 524 525 DESC_ENC(desc, MDST_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 526 DESC_ENC(desc, length_div_16, MDST_LENGTH_SHIFT, MDST_LENGTH_MASK); 527 DESC_ENC(desc, addr, MDST_ADDR_SHIFT, MDST_ADDR_MASK); 528 529 return desc; 530 } 531 532 static u64 flexrm_imm_desc(u64 data) 533 { 534 u64 desc = 0; 535 536 DESC_ENC(desc, IMM_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 537 DESC_ENC(desc, data, IMM_DATA_SHIFT, IMM_DATA_MASK); 538 539 return desc; 540 } 541 542 static u64 flexrm_srct_desc(dma_addr_t addr, unsigned int length) 543 { 544 u64 desc = 0; 545 546 DESC_ENC(desc, SRCT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 547 DESC_ENC(desc, length, SRCT_LENGTH_SHIFT, SRCT_LENGTH_MASK); 548 DESC_ENC(desc, addr, SRCT_ADDR_SHIFT, SRCT_ADDR_MASK); 549 550 return desc; 551 } 552 553 static u64 flexrm_dstt_desc(dma_addr_t addr, unsigned int length) 554 { 555 u64 desc = 0; 556 557 DESC_ENC(desc, DSTT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 558 DESC_ENC(desc, length, DSTT_LENGTH_SHIFT, DSTT_LENGTH_MASK); 559 DESC_ENC(desc, addr, DSTT_ADDR_SHIFT, DSTT_ADDR_MASK); 560 561 return desc; 562 } 563 564 static u64 flexrm_immt_desc(u64 data) 565 { 566 u64 desc = 0; 567 568 DESC_ENC(desc, IMMT_TYPE, DESC_TYPE_SHIFT, DESC_TYPE_MASK); 569 DESC_ENC(desc, data, IMMT_DATA_SHIFT, IMMT_DATA_MASK); 570 571 return desc; 572 } 573 574 static bool flexrm_spu_sanity_check(struct brcm_message *msg) 575 { 576 struct scatterlist *sg; 577 578 if (!msg->spu.src || !msg->spu.dst) 579 return false; 580 for (sg = msg->spu.src; sg; sg = sg_next(sg)) { 581 if (sg->length & 0xf) { 582 if (sg->length > SRC_LENGTH_MASK) 583 return false; 584 } else { 585 if (sg->length > (MSRC_LENGTH_MASK * 16)) 586 return false; 587 } 588 } 589 for (sg = msg->spu.dst; sg; sg = sg_next(sg)) { 590 if (sg->length & 0xf) { 591 if (sg->length > DST_LENGTH_MASK) 592 return false; 593 } else { 594 if (sg->length > (MDST_LENGTH_MASK * 16)) 595 return false; 596 } 597 } 598 599 return true; 600 } 601 602 static u32 flexrm_spu_estimate_nonheader_desc_count(struct brcm_message *msg) 603 { 604 u32 cnt = 0; 605 unsigned int dst_target = 0; 606 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst; 607 608 while (src_sg || dst_sg) { 609 if (src_sg) { 610 cnt++; 611 dst_target = src_sg->length; 612 src_sg = sg_next(src_sg); 613 } else 614 dst_target = UINT_MAX; 615 616 while (dst_target && dst_sg) { 617 cnt++; 618 if (dst_sg->length < dst_target) 619 dst_target -= dst_sg->length; 620 else 621 dst_target = 0; 622 dst_sg = sg_next(dst_sg); 623 } 624 } 625 626 return cnt; 627 } 628 629 static int flexrm_spu_dma_map(struct device *dev, struct brcm_message *msg) 630 { 631 int rc; 632 633 rc = dma_map_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 634 DMA_TO_DEVICE); 635 if (rc < 0) 636 return rc; 637 638 rc = dma_map_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), 639 DMA_FROM_DEVICE); 640 if (rc < 0) { 641 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 642 DMA_TO_DEVICE); 643 return rc; 644 } 645 646 return 0; 647 } 648 649 static void flexrm_spu_dma_unmap(struct device *dev, struct brcm_message *msg) 650 { 651 dma_unmap_sg(dev, msg->spu.dst, sg_nents(msg->spu.dst), 652 DMA_FROM_DEVICE); 653 dma_unmap_sg(dev, msg->spu.src, sg_nents(msg->spu.src), 654 DMA_TO_DEVICE); 655 } 656 657 static void *flexrm_spu_write_descs(struct brcm_message *msg, u32 nhcnt, 658 u32 reqid, void *desc_ptr, u32 toggle, 659 void *start_desc, void *end_desc) 660 { 661 u64 d; 662 u32 nhpos = 0; 663 void *orig_desc_ptr = desc_ptr; 664 unsigned int dst_target = 0; 665 struct scatterlist *src_sg = msg->spu.src, *dst_sg = msg->spu.dst; 666 667 while (src_sg || dst_sg) { 668 if (src_sg) { 669 if (sg_dma_len(src_sg) & 0xf) 670 d = flexrm_src_desc(sg_dma_address(src_sg), 671 sg_dma_len(src_sg)); 672 else 673 d = flexrm_msrc_desc(sg_dma_address(src_sg), 674 sg_dma_len(src_sg)/16); 675 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 676 d, &desc_ptr, &toggle, 677 start_desc, end_desc); 678 nhpos++; 679 dst_target = sg_dma_len(src_sg); 680 src_sg = sg_next(src_sg); 681 } else 682 dst_target = UINT_MAX; 683 684 while (dst_target && dst_sg) { 685 if (sg_dma_len(dst_sg) & 0xf) 686 d = flexrm_dst_desc(sg_dma_address(dst_sg), 687 sg_dma_len(dst_sg)); 688 else 689 d = flexrm_mdst_desc(sg_dma_address(dst_sg), 690 sg_dma_len(dst_sg)/16); 691 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 692 d, &desc_ptr, &toggle, 693 start_desc, end_desc); 694 nhpos++; 695 if (sg_dma_len(dst_sg) < dst_target) 696 dst_target -= sg_dma_len(dst_sg); 697 else 698 dst_target = 0; 699 dst_sg = sg_next(dst_sg); 700 } 701 } 702 703 /* Null descriptor with invalid toggle bit */ 704 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle)); 705 706 /* Ensure that descriptors have been written to memory */ 707 wmb(); 708 709 /* Flip toggle bit in header */ 710 flexrm_flip_header_toggle(orig_desc_ptr); 711 712 return desc_ptr; 713 } 714 715 static bool flexrm_sba_sanity_check(struct brcm_message *msg) 716 { 717 u32 i; 718 719 if (!msg->sba.cmds || !msg->sba.cmds_count) 720 return false; 721 722 for (i = 0; i < msg->sba.cmds_count; i++) { 723 if (((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) || 724 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) && 725 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT)) 726 return false; 727 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) && 728 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK)) 729 return false; 730 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C) && 731 (msg->sba.cmds[i].data_len > SRCT_LENGTH_MASK)) 732 return false; 733 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) && 734 (msg->sba.cmds[i].resp_len > DSTT_LENGTH_MASK)) 735 return false; 736 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) && 737 (msg->sba.cmds[i].data_len > DSTT_LENGTH_MASK)) 738 return false; 739 } 740 741 return true; 742 } 743 744 static u32 flexrm_sba_estimate_nonheader_desc_count(struct brcm_message *msg) 745 { 746 u32 i, cnt; 747 748 cnt = 0; 749 for (i = 0; i < msg->sba.cmds_count; i++) { 750 cnt++; 751 752 if ((msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_B) || 753 (msg->sba.cmds[i].flags & BRCM_SBA_CMD_TYPE_C)) 754 cnt++; 755 756 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_RESP) 757 cnt++; 758 759 if (msg->sba.cmds[i].flags & BRCM_SBA_CMD_HAS_OUTPUT) 760 cnt++; 761 } 762 763 return cnt; 764 } 765 766 static void *flexrm_sba_write_descs(struct brcm_message *msg, u32 nhcnt, 767 u32 reqid, void *desc_ptr, u32 toggle, 768 void *start_desc, void *end_desc) 769 { 770 u64 d; 771 u32 i, nhpos = 0; 772 struct brcm_sba_command *c; 773 void *orig_desc_ptr = desc_ptr; 774 775 /* Convert SBA commands into descriptors */ 776 for (i = 0; i < msg->sba.cmds_count; i++) { 777 c = &msg->sba.cmds[i]; 778 779 if ((c->flags & BRCM_SBA_CMD_HAS_RESP) && 780 (c->flags & BRCM_SBA_CMD_HAS_OUTPUT)) { 781 /* Destination response descriptor */ 782 d = flexrm_dst_desc(c->resp, c->resp_len); 783 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 784 d, &desc_ptr, &toggle, 785 start_desc, end_desc); 786 nhpos++; 787 } else if (c->flags & BRCM_SBA_CMD_HAS_RESP) { 788 /* Destination response with tlast descriptor */ 789 d = flexrm_dstt_desc(c->resp, c->resp_len); 790 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 791 d, &desc_ptr, &toggle, 792 start_desc, end_desc); 793 nhpos++; 794 } 795 796 if (c->flags & BRCM_SBA_CMD_HAS_OUTPUT) { 797 /* Destination with tlast descriptor */ 798 d = flexrm_dstt_desc(c->data, c->data_len); 799 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 800 d, &desc_ptr, &toggle, 801 start_desc, end_desc); 802 nhpos++; 803 } 804 805 if (c->flags & BRCM_SBA_CMD_TYPE_B) { 806 /* Command as immediate descriptor */ 807 d = flexrm_imm_desc(c->cmd); 808 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 809 d, &desc_ptr, &toggle, 810 start_desc, end_desc); 811 nhpos++; 812 } else { 813 /* Command as immediate descriptor with tlast */ 814 d = flexrm_immt_desc(c->cmd); 815 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 816 d, &desc_ptr, &toggle, 817 start_desc, end_desc); 818 nhpos++; 819 } 820 821 if ((c->flags & BRCM_SBA_CMD_TYPE_B) || 822 (c->flags & BRCM_SBA_CMD_TYPE_C)) { 823 /* Source with tlast descriptor */ 824 d = flexrm_srct_desc(c->data, c->data_len); 825 flexrm_enqueue_desc(nhpos, nhcnt, reqid, 826 d, &desc_ptr, &toggle, 827 start_desc, end_desc); 828 nhpos++; 829 } 830 } 831 832 /* Null descriptor with invalid toggle bit */ 833 flexrm_write_desc(desc_ptr, flexrm_null_desc(!toggle)); 834 835 /* Ensure that descriptors have been written to memory */ 836 wmb(); 837 838 /* Flip toggle bit in header */ 839 flexrm_flip_header_toggle(orig_desc_ptr); 840 841 return desc_ptr; 842 } 843 844 static bool flexrm_sanity_check(struct brcm_message *msg) 845 { 846 if (!msg) 847 return false; 848 849 switch (msg->type) { 850 case BRCM_MESSAGE_SPU: 851 return flexrm_spu_sanity_check(msg); 852 case BRCM_MESSAGE_SBA: 853 return flexrm_sba_sanity_check(msg); 854 default: 855 return false; 856 }; 857 } 858 859 static u32 flexrm_estimate_nonheader_desc_count(struct brcm_message *msg) 860 { 861 if (!msg) 862 return 0; 863 864 switch (msg->type) { 865 case BRCM_MESSAGE_SPU: 866 return flexrm_spu_estimate_nonheader_desc_count(msg); 867 case BRCM_MESSAGE_SBA: 868 return flexrm_sba_estimate_nonheader_desc_count(msg); 869 default: 870 return 0; 871 }; 872 } 873 874 static int flexrm_dma_map(struct device *dev, struct brcm_message *msg) 875 { 876 if (!dev || !msg) 877 return -EINVAL; 878 879 switch (msg->type) { 880 case BRCM_MESSAGE_SPU: 881 return flexrm_spu_dma_map(dev, msg); 882 default: 883 break; 884 } 885 886 return 0; 887 } 888 889 static void flexrm_dma_unmap(struct device *dev, struct brcm_message *msg) 890 { 891 if (!dev || !msg) 892 return; 893 894 switch (msg->type) { 895 case BRCM_MESSAGE_SPU: 896 flexrm_spu_dma_unmap(dev, msg); 897 break; 898 default: 899 break; 900 } 901 } 902 903 static void *flexrm_write_descs(struct brcm_message *msg, u32 nhcnt, 904 u32 reqid, void *desc_ptr, u32 toggle, 905 void *start_desc, void *end_desc) 906 { 907 if (!msg || !desc_ptr || !start_desc || !end_desc) 908 return ERR_PTR(-ENOTSUPP); 909 910 if ((desc_ptr < start_desc) || (end_desc <= desc_ptr)) 911 return ERR_PTR(-ERANGE); 912 913 switch (msg->type) { 914 case BRCM_MESSAGE_SPU: 915 return flexrm_spu_write_descs(msg, nhcnt, reqid, 916 desc_ptr, toggle, 917 start_desc, end_desc); 918 case BRCM_MESSAGE_SBA: 919 return flexrm_sba_write_descs(msg, nhcnt, reqid, 920 desc_ptr, toggle, 921 start_desc, end_desc); 922 default: 923 return ERR_PTR(-ENOTSUPP); 924 }; 925 } 926 927 /* ====== FlexRM driver helper routines ===== */ 928 929 static void flexrm_write_config_in_seqfile(struct flexrm_mbox *mbox, 930 struct seq_file *file) 931 { 932 int i; 933 const char *state; 934 struct flexrm_ring *ring; 935 936 seq_printf(file, "%-5s %-9s %-18s %-10s %-18s %-10s\n", 937 "Ring#", "State", "BD_Addr", "BD_Size", 938 "Cmpl_Addr", "Cmpl_Size"); 939 940 for (i = 0; i < mbox->num_rings; i++) { 941 ring = &mbox->rings[i]; 942 if (readl(ring->regs + RING_CONTROL) & 943 BIT(CONTROL_ACTIVE_SHIFT)) 944 state = "active"; 945 else 946 state = "inactive"; 947 seq_printf(file, 948 "%-5d %-9s 0x%016llx 0x%08x 0x%016llx 0x%08x\n", 949 ring->num, state, 950 (unsigned long long)ring->bd_dma_base, 951 (u32)RING_BD_SIZE, 952 (unsigned long long)ring->cmpl_dma_base, 953 (u32)RING_CMPL_SIZE); 954 } 955 } 956 957 static void flexrm_write_stats_in_seqfile(struct flexrm_mbox *mbox, 958 struct seq_file *file) 959 { 960 int i; 961 u32 val, bd_read_offset; 962 struct flexrm_ring *ring; 963 964 seq_printf(file, "%-5s %-10s %-10s %-10s %-11s %-11s\n", 965 "Ring#", "BD_Read", "BD_Write", 966 "Cmpl_Read", "Submitted", "Completed"); 967 968 for (i = 0; i < mbox->num_rings; i++) { 969 ring = &mbox->rings[i]; 970 bd_read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR); 971 val = readl_relaxed(ring->regs + RING_BD_START_ADDR); 972 bd_read_offset *= RING_DESC_SIZE; 973 bd_read_offset += (u32)(BD_START_ADDR_DECODE(val) - 974 ring->bd_dma_base); 975 seq_printf(file, "%-5d 0x%08x 0x%08x 0x%08x %-11d %-11d\n", 976 ring->num, 977 (u32)bd_read_offset, 978 (u32)ring->bd_write_offset, 979 (u32)ring->cmpl_read_offset, 980 (u32)atomic_read(&ring->msg_send_count), 981 (u32)atomic_read(&ring->msg_cmpl_count)); 982 } 983 } 984 985 static int flexrm_new_request(struct flexrm_ring *ring, 986 struct brcm_message *batch_msg, 987 struct brcm_message *msg) 988 { 989 void *next; 990 unsigned long flags; 991 u32 val, count, nhcnt; 992 u32 read_offset, write_offset; 993 bool exit_cleanup = false; 994 int ret = 0, reqid; 995 996 /* Do sanity check on message */ 997 if (!flexrm_sanity_check(msg)) 998 return -EIO; 999 msg->error = 0; 1000 1001 /* If no requests possible then save data pointer and goto done. */ 1002 spin_lock_irqsave(&ring->lock, flags); 1003 reqid = bitmap_find_free_region(ring->requests_bmap, 1004 RING_MAX_REQ_COUNT, 0); 1005 spin_unlock_irqrestore(&ring->lock, flags); 1006 if (reqid < 0) 1007 return -ENOSPC; 1008 ring->requests[reqid] = msg; 1009 1010 /* Do DMA mappings for the message */ 1011 ret = flexrm_dma_map(ring->mbox->dev, msg); 1012 if (ret < 0) { 1013 ring->requests[reqid] = NULL; 1014 spin_lock_irqsave(&ring->lock, flags); 1015 bitmap_release_region(ring->requests_bmap, reqid, 0); 1016 spin_unlock_irqrestore(&ring->lock, flags); 1017 return ret; 1018 } 1019 1020 /* Determine current HW BD read offset */ 1021 read_offset = readl_relaxed(ring->regs + RING_BD_READ_PTR); 1022 val = readl_relaxed(ring->regs + RING_BD_START_ADDR); 1023 read_offset *= RING_DESC_SIZE; 1024 read_offset += (u32)(BD_START_ADDR_DECODE(val) - ring->bd_dma_base); 1025 1026 /* 1027 * Number required descriptors = number of non-header descriptors + 1028 * number of header descriptors + 1029 * 1x null descriptor 1030 */ 1031 nhcnt = flexrm_estimate_nonheader_desc_count(msg); 1032 count = flexrm_estimate_header_desc_count(nhcnt) + nhcnt + 1; 1033 1034 /* Check for available descriptor space. */ 1035 write_offset = ring->bd_write_offset; 1036 while (count) { 1037 if (!flexrm_is_next_table_desc(ring->bd_base + write_offset)) 1038 count--; 1039 write_offset += RING_DESC_SIZE; 1040 if (write_offset == RING_BD_SIZE) 1041 write_offset = 0x0; 1042 if (write_offset == read_offset) 1043 break; 1044 } 1045 if (count) { 1046 ret = -ENOSPC; 1047 exit_cleanup = true; 1048 goto exit; 1049 } 1050 1051 /* Write descriptors to ring */ 1052 next = flexrm_write_descs(msg, nhcnt, reqid, 1053 ring->bd_base + ring->bd_write_offset, 1054 RING_BD_TOGGLE_VALID(ring->bd_write_offset), 1055 ring->bd_base, ring->bd_base + RING_BD_SIZE); 1056 if (IS_ERR(next)) { 1057 ret = PTR_ERR(next); 1058 exit_cleanup = true; 1059 goto exit; 1060 } 1061 1062 /* Save ring BD write offset */ 1063 ring->bd_write_offset = (unsigned long)(next - ring->bd_base); 1064 1065 /* Increment number of messages sent */ 1066 atomic_inc_return(&ring->msg_send_count); 1067 1068 exit: 1069 /* Update error status in message */ 1070 msg->error = ret; 1071 1072 /* Cleanup if we failed */ 1073 if (exit_cleanup) { 1074 flexrm_dma_unmap(ring->mbox->dev, msg); 1075 ring->requests[reqid] = NULL; 1076 spin_lock_irqsave(&ring->lock, flags); 1077 bitmap_release_region(ring->requests_bmap, reqid, 0); 1078 spin_unlock_irqrestore(&ring->lock, flags); 1079 } 1080 1081 return ret; 1082 } 1083 1084 static int flexrm_process_completions(struct flexrm_ring *ring) 1085 { 1086 u64 desc; 1087 int err, count = 0; 1088 unsigned long flags; 1089 struct brcm_message *msg = NULL; 1090 u32 reqid, cmpl_read_offset, cmpl_write_offset; 1091 struct mbox_chan *chan = &ring->mbox->controller.chans[ring->num]; 1092 1093 spin_lock_irqsave(&ring->lock, flags); 1094 1095 /* 1096 * Get current completion read and write offset 1097 * 1098 * Note: We should read completion write pointer atleast once 1099 * after we get a MSI interrupt because HW maintains internal 1100 * MSI status which will allow next MSI interrupt only after 1101 * completion write pointer is read. 1102 */ 1103 cmpl_write_offset = readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR); 1104 cmpl_write_offset *= RING_DESC_SIZE; 1105 cmpl_read_offset = ring->cmpl_read_offset; 1106 ring->cmpl_read_offset = cmpl_write_offset; 1107 1108 spin_unlock_irqrestore(&ring->lock, flags); 1109 1110 /* For each completed request notify mailbox clients */ 1111 reqid = 0; 1112 while (cmpl_read_offset != cmpl_write_offset) { 1113 /* Dequeue next completion descriptor */ 1114 desc = *((u64 *)(ring->cmpl_base + cmpl_read_offset)); 1115 1116 /* Next read offset */ 1117 cmpl_read_offset += RING_DESC_SIZE; 1118 if (cmpl_read_offset == RING_CMPL_SIZE) 1119 cmpl_read_offset = 0; 1120 1121 /* Decode error from completion descriptor */ 1122 err = flexrm_cmpl_desc_to_error(desc); 1123 if (err < 0) { 1124 dev_warn(ring->mbox->dev, 1125 "ring%d got completion desc=0x%lx with error %d\n", 1126 ring->num, (unsigned long)desc, err); 1127 } 1128 1129 /* Determine request id from completion descriptor */ 1130 reqid = flexrm_cmpl_desc_to_reqid(desc); 1131 1132 /* Determine message pointer based on reqid */ 1133 msg = ring->requests[reqid]; 1134 if (!msg) { 1135 dev_warn(ring->mbox->dev, 1136 "ring%d null msg pointer for completion desc=0x%lx\n", 1137 ring->num, (unsigned long)desc); 1138 continue; 1139 } 1140 1141 /* Release reqid for recycling */ 1142 ring->requests[reqid] = NULL; 1143 spin_lock_irqsave(&ring->lock, flags); 1144 bitmap_release_region(ring->requests_bmap, reqid, 0); 1145 spin_unlock_irqrestore(&ring->lock, flags); 1146 1147 /* Unmap DMA mappings */ 1148 flexrm_dma_unmap(ring->mbox->dev, msg); 1149 1150 /* Give-back message to mailbox client */ 1151 msg->error = err; 1152 mbox_chan_received_data(chan, msg); 1153 1154 /* Increment number of completions processed */ 1155 atomic_inc_return(&ring->msg_cmpl_count); 1156 count++; 1157 } 1158 1159 return count; 1160 } 1161 1162 /* ====== FlexRM Debugfs callbacks ====== */ 1163 1164 static int flexrm_debugfs_conf_show(struct seq_file *file, void *offset) 1165 { 1166 struct platform_device *pdev = to_platform_device(file->private); 1167 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1168 1169 /* Write config in file */ 1170 flexrm_write_config_in_seqfile(mbox, file); 1171 1172 return 0; 1173 } 1174 1175 static int flexrm_debugfs_stats_show(struct seq_file *file, void *offset) 1176 { 1177 struct platform_device *pdev = to_platform_device(file->private); 1178 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1179 1180 /* Write stats in file */ 1181 flexrm_write_stats_in_seqfile(mbox, file); 1182 1183 return 0; 1184 } 1185 1186 /* ====== FlexRM interrupt handler ===== */ 1187 1188 static irqreturn_t flexrm_irq_event(int irq, void *dev_id) 1189 { 1190 /* We only have MSI for completions so just wakeup IRQ thread */ 1191 /* Ring related errors will be informed via completion descriptors */ 1192 1193 return IRQ_WAKE_THREAD; 1194 } 1195 1196 static irqreturn_t flexrm_irq_thread(int irq, void *dev_id) 1197 { 1198 flexrm_process_completions(dev_id); 1199 1200 return IRQ_HANDLED; 1201 } 1202 1203 /* ====== FlexRM mailbox callbacks ===== */ 1204 1205 static int flexrm_send_data(struct mbox_chan *chan, void *data) 1206 { 1207 int i, rc; 1208 struct flexrm_ring *ring = chan->con_priv; 1209 struct brcm_message *msg = data; 1210 1211 if (msg->type == BRCM_MESSAGE_BATCH) { 1212 for (i = msg->batch.msgs_queued; 1213 i < msg->batch.msgs_count; i++) { 1214 rc = flexrm_new_request(ring, msg, 1215 &msg->batch.msgs[i]); 1216 if (rc) { 1217 msg->error = rc; 1218 return rc; 1219 } 1220 msg->batch.msgs_queued++; 1221 } 1222 return 0; 1223 } 1224 1225 return flexrm_new_request(ring, NULL, data); 1226 } 1227 1228 static bool flexrm_peek_data(struct mbox_chan *chan) 1229 { 1230 int cnt = flexrm_process_completions(chan->con_priv); 1231 1232 return (cnt > 0) ? true : false; 1233 } 1234 1235 static int flexrm_startup(struct mbox_chan *chan) 1236 { 1237 u64 d; 1238 u32 val, off; 1239 int ret = 0; 1240 dma_addr_t next_addr; 1241 struct flexrm_ring *ring = chan->con_priv; 1242 1243 /* Allocate BD memory */ 1244 ring->bd_base = dma_pool_alloc(ring->mbox->bd_pool, 1245 GFP_KERNEL, &ring->bd_dma_base); 1246 if (!ring->bd_base) { 1247 dev_err(ring->mbox->dev, 1248 "can't allocate BD memory for ring%d\n", 1249 ring->num); 1250 ret = -ENOMEM; 1251 goto fail; 1252 } 1253 1254 /* Configure next table pointer entries in BD memory */ 1255 for (off = 0; off < RING_BD_SIZE; off += RING_DESC_SIZE) { 1256 next_addr = off + RING_DESC_SIZE; 1257 if (next_addr == RING_BD_SIZE) 1258 next_addr = 0; 1259 next_addr += ring->bd_dma_base; 1260 if (RING_BD_ALIGN_CHECK(next_addr)) 1261 d = flexrm_next_table_desc(RING_BD_TOGGLE_VALID(off), 1262 next_addr); 1263 else 1264 d = flexrm_null_desc(RING_BD_TOGGLE_INVALID(off)); 1265 flexrm_write_desc(ring->bd_base + off, d); 1266 } 1267 1268 /* Allocate completion memory */ 1269 ring->cmpl_base = dma_pool_zalloc(ring->mbox->cmpl_pool, 1270 GFP_KERNEL, &ring->cmpl_dma_base); 1271 if (!ring->cmpl_base) { 1272 dev_err(ring->mbox->dev, 1273 "can't allocate completion memory for ring%d\n", 1274 ring->num); 1275 ret = -ENOMEM; 1276 goto fail_free_bd_memory; 1277 } 1278 1279 /* Request IRQ */ 1280 if (ring->irq == UINT_MAX) { 1281 dev_err(ring->mbox->dev, 1282 "ring%d IRQ not available\n", ring->num); 1283 ret = -ENODEV; 1284 goto fail_free_cmpl_memory; 1285 } 1286 ret = request_threaded_irq(ring->irq, 1287 flexrm_irq_event, 1288 flexrm_irq_thread, 1289 0, dev_name(ring->mbox->dev), ring); 1290 if (ret) { 1291 dev_err(ring->mbox->dev, 1292 "failed to request ring%d IRQ\n", ring->num); 1293 goto fail_free_cmpl_memory; 1294 } 1295 ring->irq_requested = true; 1296 1297 /* Set IRQ affinity hint */ 1298 ring->irq_aff_hint = CPU_MASK_NONE; 1299 val = ring->mbox->num_rings; 1300 val = (num_online_cpus() < val) ? val / num_online_cpus() : 1; 1301 cpumask_set_cpu((ring->num / val) % num_online_cpus(), 1302 &ring->irq_aff_hint); 1303 ret = irq_set_affinity_hint(ring->irq, &ring->irq_aff_hint); 1304 if (ret) { 1305 dev_err(ring->mbox->dev, 1306 "failed to set IRQ affinity hint for ring%d\n", 1307 ring->num); 1308 goto fail_free_irq; 1309 } 1310 1311 /* Disable/inactivate ring */ 1312 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1313 1314 /* Program BD start address */ 1315 val = BD_START_ADDR_VALUE(ring->bd_dma_base); 1316 writel_relaxed(val, ring->regs + RING_BD_START_ADDR); 1317 1318 /* BD write pointer will be same as HW write pointer */ 1319 ring->bd_write_offset = 1320 readl_relaxed(ring->regs + RING_BD_WRITE_PTR); 1321 ring->bd_write_offset *= RING_DESC_SIZE; 1322 1323 /* Program completion start address */ 1324 val = CMPL_START_ADDR_VALUE(ring->cmpl_dma_base); 1325 writel_relaxed(val, ring->regs + RING_CMPL_START_ADDR); 1326 1327 /* Completion read pointer will be same as HW write pointer */ 1328 ring->cmpl_read_offset = 1329 readl_relaxed(ring->regs + RING_CMPL_WRITE_PTR); 1330 ring->cmpl_read_offset *= RING_DESC_SIZE; 1331 1332 /* Read ring Tx, Rx, and Outstanding counts to clear */ 1333 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_LS); 1334 readl_relaxed(ring->regs + RING_NUM_REQ_RECV_MS); 1335 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_LS); 1336 readl_relaxed(ring->regs + RING_NUM_REQ_TRANS_MS); 1337 readl_relaxed(ring->regs + RING_NUM_REQ_OUTSTAND); 1338 1339 /* Configure RING_MSI_CONTROL */ 1340 val = 0; 1341 val |= (ring->msi_timer_val << MSI_TIMER_VAL_SHIFT); 1342 val |= BIT(MSI_ENABLE_SHIFT); 1343 val |= (ring->msi_count_threshold & MSI_COUNT_MASK) << MSI_COUNT_SHIFT; 1344 writel_relaxed(val, ring->regs + RING_MSI_CONTROL); 1345 1346 /* Enable/activate ring */ 1347 val = BIT(CONTROL_ACTIVE_SHIFT); 1348 writel_relaxed(val, ring->regs + RING_CONTROL); 1349 1350 /* Reset stats to zero */ 1351 atomic_set(&ring->msg_send_count, 0); 1352 atomic_set(&ring->msg_cmpl_count, 0); 1353 1354 return 0; 1355 1356 fail_free_irq: 1357 free_irq(ring->irq, ring); 1358 ring->irq_requested = false; 1359 fail_free_cmpl_memory: 1360 dma_pool_free(ring->mbox->cmpl_pool, 1361 ring->cmpl_base, ring->cmpl_dma_base); 1362 ring->cmpl_base = NULL; 1363 fail_free_bd_memory: 1364 dma_pool_free(ring->mbox->bd_pool, 1365 ring->bd_base, ring->bd_dma_base); 1366 ring->bd_base = NULL; 1367 fail: 1368 return ret; 1369 } 1370 1371 static void flexrm_shutdown(struct mbox_chan *chan) 1372 { 1373 u32 reqid; 1374 unsigned int timeout; 1375 struct brcm_message *msg; 1376 struct flexrm_ring *ring = chan->con_priv; 1377 1378 /* Disable/inactivate ring */ 1379 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1380 1381 /* Set ring flush state */ 1382 timeout = 1000; /* timeout of 1s */ 1383 writel_relaxed(BIT(CONTROL_FLUSH_SHIFT), 1384 ring->regs + RING_CONTROL); 1385 do { 1386 if (readl_relaxed(ring->regs + RING_FLUSH_DONE) & 1387 FLUSH_DONE_MASK) 1388 break; 1389 mdelay(1); 1390 } while (--timeout); 1391 if (!timeout) 1392 dev_err(ring->mbox->dev, 1393 "setting ring%d flush state timedout\n", ring->num); 1394 1395 /* Clear ring flush state */ 1396 timeout = 1000; /* timeout of 1s */ 1397 writel_relaxed(0x0, ring->regs + RING_CONTROL); 1398 do { 1399 if (!(readl_relaxed(ring->regs + RING_FLUSH_DONE) & 1400 FLUSH_DONE_MASK)) 1401 break; 1402 mdelay(1); 1403 } while (--timeout); 1404 if (!timeout) 1405 dev_err(ring->mbox->dev, 1406 "clearing ring%d flush state timedout\n", ring->num); 1407 1408 /* Abort all in-flight requests */ 1409 for (reqid = 0; reqid < RING_MAX_REQ_COUNT; reqid++) { 1410 msg = ring->requests[reqid]; 1411 if (!msg) 1412 continue; 1413 1414 /* Release reqid for recycling */ 1415 ring->requests[reqid] = NULL; 1416 1417 /* Unmap DMA mappings */ 1418 flexrm_dma_unmap(ring->mbox->dev, msg); 1419 1420 /* Give-back message to mailbox client */ 1421 msg->error = -EIO; 1422 mbox_chan_received_data(chan, msg); 1423 } 1424 1425 /* Clear requests bitmap */ 1426 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT); 1427 1428 /* Release IRQ */ 1429 if (ring->irq_requested) { 1430 irq_set_affinity_hint(ring->irq, NULL); 1431 free_irq(ring->irq, ring); 1432 ring->irq_requested = false; 1433 } 1434 1435 /* Free-up completion descriptor ring */ 1436 if (ring->cmpl_base) { 1437 dma_pool_free(ring->mbox->cmpl_pool, 1438 ring->cmpl_base, ring->cmpl_dma_base); 1439 ring->cmpl_base = NULL; 1440 } 1441 1442 /* Free-up BD descriptor ring */ 1443 if (ring->bd_base) { 1444 dma_pool_free(ring->mbox->bd_pool, 1445 ring->bd_base, ring->bd_dma_base); 1446 ring->bd_base = NULL; 1447 } 1448 } 1449 1450 static const struct mbox_chan_ops flexrm_mbox_chan_ops = { 1451 .send_data = flexrm_send_data, 1452 .startup = flexrm_startup, 1453 .shutdown = flexrm_shutdown, 1454 .peek_data = flexrm_peek_data, 1455 }; 1456 1457 static struct mbox_chan *flexrm_mbox_of_xlate(struct mbox_controller *cntlr, 1458 const struct of_phandle_args *pa) 1459 { 1460 struct mbox_chan *chan; 1461 struct flexrm_ring *ring; 1462 1463 if (pa->args_count < 3) 1464 return ERR_PTR(-EINVAL); 1465 1466 if (pa->args[0] >= cntlr->num_chans) 1467 return ERR_PTR(-ENOENT); 1468 1469 if (pa->args[1] > MSI_COUNT_MASK) 1470 return ERR_PTR(-EINVAL); 1471 1472 if (pa->args[2] > MSI_TIMER_VAL_MASK) 1473 return ERR_PTR(-EINVAL); 1474 1475 chan = &cntlr->chans[pa->args[0]]; 1476 ring = chan->con_priv; 1477 ring->msi_count_threshold = pa->args[1]; 1478 ring->msi_timer_val = pa->args[2]; 1479 1480 return chan; 1481 } 1482 1483 /* ====== FlexRM platform driver ===== */ 1484 1485 static void flexrm_mbox_msi_write(struct msi_desc *desc, struct msi_msg *msg) 1486 { 1487 struct device *dev = msi_desc_to_dev(desc); 1488 struct flexrm_mbox *mbox = dev_get_drvdata(dev); 1489 struct flexrm_ring *ring = &mbox->rings[desc->platform.msi_index]; 1490 1491 /* Configure per-Ring MSI registers */ 1492 writel_relaxed(msg->address_lo, ring->regs + RING_MSI_ADDR_LS); 1493 writel_relaxed(msg->address_hi, ring->regs + RING_MSI_ADDR_MS); 1494 writel_relaxed(msg->data, ring->regs + RING_MSI_DATA_VALUE); 1495 } 1496 1497 static int flexrm_mbox_probe(struct platform_device *pdev) 1498 { 1499 int index, ret = 0; 1500 void __iomem *regs; 1501 void __iomem *regs_end; 1502 struct msi_desc *desc; 1503 struct resource *iomem; 1504 struct flexrm_ring *ring; 1505 struct flexrm_mbox *mbox; 1506 struct device *dev = &pdev->dev; 1507 1508 /* Allocate driver mailbox struct */ 1509 mbox = devm_kzalloc(dev, sizeof(*mbox), GFP_KERNEL); 1510 if (!mbox) { 1511 ret = -ENOMEM; 1512 goto fail; 1513 } 1514 mbox->dev = dev; 1515 platform_set_drvdata(pdev, mbox); 1516 1517 /* Get resource for registers */ 1518 iomem = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1519 if (!iomem || (resource_size(iomem) < RING_REGS_SIZE)) { 1520 ret = -ENODEV; 1521 goto fail; 1522 } 1523 1524 /* Map registers of all rings */ 1525 mbox->regs = devm_ioremap_resource(&pdev->dev, iomem); 1526 if (IS_ERR(mbox->regs)) { 1527 ret = PTR_ERR(mbox->regs); 1528 dev_err(&pdev->dev, "Failed to remap mailbox regs: %d\n", ret); 1529 goto fail; 1530 } 1531 regs_end = mbox->regs + resource_size(iomem); 1532 1533 /* Scan and count available rings */ 1534 mbox->num_rings = 0; 1535 for (regs = mbox->regs; regs < regs_end; regs += RING_REGS_SIZE) { 1536 if (readl_relaxed(regs + RING_VER) == RING_VER_MAGIC) 1537 mbox->num_rings++; 1538 } 1539 if (!mbox->num_rings) { 1540 ret = -ENODEV; 1541 goto fail; 1542 } 1543 1544 /* Allocate driver ring structs */ 1545 ring = devm_kcalloc(dev, mbox->num_rings, sizeof(*ring), GFP_KERNEL); 1546 if (!ring) { 1547 ret = -ENOMEM; 1548 goto fail; 1549 } 1550 mbox->rings = ring; 1551 1552 /* Initialize members of driver ring structs */ 1553 regs = mbox->regs; 1554 for (index = 0; index < mbox->num_rings; index++) { 1555 ring = &mbox->rings[index]; 1556 ring->num = index; 1557 ring->mbox = mbox; 1558 while ((regs < regs_end) && 1559 (readl_relaxed(regs + RING_VER) != RING_VER_MAGIC)) 1560 regs += RING_REGS_SIZE; 1561 if (regs_end <= regs) { 1562 ret = -ENODEV; 1563 goto fail; 1564 } 1565 ring->regs = regs; 1566 regs += RING_REGS_SIZE; 1567 ring->irq = UINT_MAX; 1568 ring->irq_requested = false; 1569 ring->msi_timer_val = MSI_TIMER_VAL_MASK; 1570 ring->msi_count_threshold = 0x1; 1571 memset(ring->requests, 0, sizeof(ring->requests)); 1572 ring->bd_base = NULL; 1573 ring->bd_dma_base = 0; 1574 ring->cmpl_base = NULL; 1575 ring->cmpl_dma_base = 0; 1576 atomic_set(&ring->msg_send_count, 0); 1577 atomic_set(&ring->msg_cmpl_count, 0); 1578 spin_lock_init(&ring->lock); 1579 bitmap_zero(ring->requests_bmap, RING_MAX_REQ_COUNT); 1580 ring->cmpl_read_offset = 0; 1581 } 1582 1583 /* FlexRM is capable of 40-bit physical addresses only */ 1584 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(40)); 1585 if (ret) { 1586 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32)); 1587 if (ret) 1588 goto fail; 1589 } 1590 1591 /* Create DMA pool for ring BD memory */ 1592 mbox->bd_pool = dma_pool_create("bd", dev, RING_BD_SIZE, 1593 1 << RING_BD_ALIGN_ORDER, 0); 1594 if (!mbox->bd_pool) { 1595 ret = -ENOMEM; 1596 goto fail; 1597 } 1598 1599 /* Create DMA pool for ring completion memory */ 1600 mbox->cmpl_pool = dma_pool_create("cmpl", dev, RING_CMPL_SIZE, 1601 1 << RING_CMPL_ALIGN_ORDER, 0); 1602 if (!mbox->cmpl_pool) { 1603 ret = -ENOMEM; 1604 } 1605 1606 /* Allocate platform MSIs for each ring */ 1607 ret = platform_msi_domain_alloc_irqs(dev, mbox->num_rings, 1608 flexrm_mbox_msi_write); 1609 if (ret) 1610 goto fail_destroy_cmpl_pool; 1611 1612 /* Save alloced IRQ numbers for each ring */ 1613 for_each_msi_entry(desc, dev) { 1614 ring = &mbox->rings[desc->platform.msi_index]; 1615 ring->irq = desc->irq; 1616 } 1617 1618 /* Check availability of debugfs */ 1619 if (!debugfs_initialized()) 1620 goto skip_debugfs; 1621 1622 /* Create debugfs root entry */ 1623 mbox->root = debugfs_create_dir(dev_name(mbox->dev), NULL); 1624 1625 /* Create debugfs config entry */ 1626 debugfs_create_devm_seqfile(mbox->dev, "config", mbox->root, 1627 flexrm_debugfs_conf_show); 1628 1629 /* Create debugfs stats entry */ 1630 debugfs_create_devm_seqfile(mbox->dev, "stats", mbox->root, 1631 flexrm_debugfs_stats_show); 1632 1633 skip_debugfs: 1634 1635 /* Initialize mailbox controller */ 1636 mbox->controller.txdone_irq = false; 1637 mbox->controller.txdone_poll = false; 1638 mbox->controller.ops = &flexrm_mbox_chan_ops; 1639 mbox->controller.dev = dev; 1640 mbox->controller.num_chans = mbox->num_rings; 1641 mbox->controller.of_xlate = flexrm_mbox_of_xlate; 1642 mbox->controller.chans = devm_kcalloc(dev, mbox->num_rings, 1643 sizeof(*mbox->controller.chans), GFP_KERNEL); 1644 if (!mbox->controller.chans) { 1645 ret = -ENOMEM; 1646 goto fail_free_debugfs_root; 1647 } 1648 for (index = 0; index < mbox->num_rings; index++) 1649 mbox->controller.chans[index].con_priv = &mbox->rings[index]; 1650 1651 /* Register mailbox controller */ 1652 ret = devm_mbox_controller_register(dev, &mbox->controller); 1653 if (ret) 1654 goto fail_free_debugfs_root; 1655 1656 dev_info(dev, "registered flexrm mailbox with %d channels\n", 1657 mbox->controller.num_chans); 1658 1659 return 0; 1660 1661 fail_free_debugfs_root: 1662 debugfs_remove_recursive(mbox->root); 1663 platform_msi_domain_free_irqs(dev); 1664 fail_destroy_cmpl_pool: 1665 dma_pool_destroy(mbox->cmpl_pool); 1666 dma_pool_destroy(mbox->bd_pool); 1667 fail: 1668 return ret; 1669 } 1670 1671 static int flexrm_mbox_remove(struct platform_device *pdev) 1672 { 1673 struct device *dev = &pdev->dev; 1674 struct flexrm_mbox *mbox = platform_get_drvdata(pdev); 1675 1676 debugfs_remove_recursive(mbox->root); 1677 1678 platform_msi_domain_free_irqs(dev); 1679 1680 dma_pool_destroy(mbox->cmpl_pool); 1681 dma_pool_destroy(mbox->bd_pool); 1682 1683 return 0; 1684 } 1685 1686 static const struct of_device_id flexrm_mbox_of_match[] = { 1687 { .compatible = "brcm,iproc-flexrm-mbox", }, 1688 {}, 1689 }; 1690 MODULE_DEVICE_TABLE(of, flexrm_mbox_of_match); 1691 1692 static struct platform_driver flexrm_mbox_driver = { 1693 .driver = { 1694 .name = "brcm-flexrm-mbox", 1695 .of_match_table = flexrm_mbox_of_match, 1696 }, 1697 .probe = flexrm_mbox_probe, 1698 .remove = flexrm_mbox_remove, 1699 }; 1700 module_platform_driver(flexrm_mbox_driver); 1701 1702 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1703 MODULE_DESCRIPTION("Broadcom FlexRM mailbox driver"); 1704 MODULE_LICENSE("GPL v2"); 1705