1 // SPDX-License-Identifier: GPL-2.0-only 2 // Copyright (C) 2017 Broadcom 3 4 /* 5 * Broadcom SBA RAID Driver 6 * 7 * The Broadcom stream buffer accelerator (SBA) provides offloading 8 * capabilities for RAID operations. The SBA offload engine is accessible 9 * via Broadcom SoC specific ring manager. Two or more offload engines 10 * can share same Broadcom SoC specific ring manager due to this Broadcom 11 * SoC specific ring manager driver is implemented as a mailbox controller 12 * driver and offload engine drivers are implemented as mallbox clients. 13 * 14 * Typically, Broadcom SoC specific ring manager will implement larger 15 * number of hardware rings over one or more SBA hardware devices. By 16 * design, the internal buffer size of SBA hardware device is limited 17 * but all offload operations supported by SBA can be broken down into 18 * multiple small size requests and executed parallelly on multiple SBA 19 * hardware devices for achieving high through-put. 20 * 21 * The Broadcom SBA RAID driver does not require any register programming 22 * except submitting request to SBA hardware device via mailbox channels. 23 * This driver implements a DMA device with one DMA channel using a single 24 * mailbox channel provided by Broadcom SoC specific ring manager driver. 25 * For having more SBA DMA channels, we can create more SBA device nodes 26 * in Broadcom SoC specific DTS based on number of hardware rings supported 27 * by Broadcom SoC ring manager. 28 */ 29 30 #include <linux/bitops.h> 31 #include <linux/debugfs.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/dmaengine.h> 34 #include <linux/list.h> 35 #include <linux/mailbox_client.h> 36 #include <linux/mailbox/brcm-message.h> 37 #include <linux/module.h> 38 #include <linux/of.h> 39 #include <linux/of_platform.h> 40 #include <linux/platform_device.h> 41 #include <linux/slab.h> 42 #include <linux/raid/pq.h> 43 #include <linux/raid/pq_tables.h> 44 45 #include "dmaengine.h" 46 47 /* ====== Driver macros and defines ===== */ 48 49 #define SBA_TYPE_SHIFT 48 50 #define SBA_TYPE_MASK GENMASK(1, 0) 51 #define SBA_TYPE_A 0x0 52 #define SBA_TYPE_B 0x2 53 #define SBA_TYPE_C 0x3 54 #define SBA_USER_DEF_SHIFT 32 55 #define SBA_USER_DEF_MASK GENMASK(15, 0) 56 #define SBA_R_MDATA_SHIFT 24 57 #define SBA_R_MDATA_MASK GENMASK(7, 0) 58 #define SBA_C_MDATA_MS_SHIFT 18 59 #define SBA_C_MDATA_MS_MASK GENMASK(1, 0) 60 #define SBA_INT_SHIFT 17 61 #define SBA_INT_MASK BIT(0) 62 #define SBA_RESP_SHIFT 16 63 #define SBA_RESP_MASK BIT(0) 64 #define SBA_C_MDATA_SHIFT 8 65 #define SBA_C_MDATA_MASK GENMASK(7, 0) 66 #define SBA_C_MDATA_BNUMx_SHIFT(__bnum) (2 * (__bnum)) 67 #define SBA_C_MDATA_BNUMx_MASK GENMASK(1, 0) 68 #define SBA_C_MDATA_DNUM_SHIFT 5 69 #define SBA_C_MDATA_DNUM_MASK GENMASK(4, 0) 70 #define SBA_C_MDATA_LS(__v) ((__v) & 0xff) 71 #define SBA_C_MDATA_MS(__v) (((__v) >> 8) & 0x3) 72 #define SBA_CMD_SHIFT 0 73 #define SBA_CMD_MASK GENMASK(3, 0) 74 #define SBA_CMD_ZERO_BUFFER 0x4 75 #define SBA_CMD_ZERO_ALL_BUFFERS 0x8 76 #define SBA_CMD_LOAD_BUFFER 0x9 77 #define SBA_CMD_XOR 0xa 78 #define SBA_CMD_GALOIS_XOR 0xb 79 #define SBA_CMD_WRITE_BUFFER 0xc 80 #define SBA_CMD_GALOIS 0xe 81 82 #define SBA_MAX_REQ_PER_MBOX_CHANNEL 8192 83 #define SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL 8 84 85 /* Driver helper macros */ 86 #define to_sba_request(tx) \ 87 container_of(tx, struct sba_request, tx) 88 #define to_sba_device(dchan) \ 89 container_of(dchan, struct sba_device, dma_chan) 90 91 /* ===== Driver data structures ===== */ 92 93 enum sba_request_flags { 94 SBA_REQUEST_STATE_FREE = 0x001, 95 SBA_REQUEST_STATE_ALLOCED = 0x002, 96 SBA_REQUEST_STATE_PENDING = 0x004, 97 SBA_REQUEST_STATE_ACTIVE = 0x008, 98 SBA_REQUEST_STATE_ABORTED = 0x010, 99 SBA_REQUEST_STATE_MASK = 0x0ff, 100 SBA_REQUEST_FENCE = 0x100, 101 }; 102 103 struct sba_request { 104 /* Global state */ 105 struct list_head node; 106 struct sba_device *sba; 107 u32 flags; 108 /* Chained requests management */ 109 struct sba_request *first; 110 struct list_head next; 111 atomic_t next_pending_count; 112 /* BRCM message data */ 113 struct brcm_message msg; 114 struct dma_async_tx_descriptor tx; 115 /* SBA commands */ 116 struct brcm_sba_command cmds[]; 117 }; 118 119 enum sba_version { 120 SBA_VER_1 = 0, 121 SBA_VER_2 122 }; 123 124 struct sba_device { 125 /* Underlying device */ 126 struct device *dev; 127 /* DT configuration parameters */ 128 enum sba_version ver; 129 /* Derived configuration parameters */ 130 u32 max_req; 131 u32 hw_buf_size; 132 u32 hw_resp_size; 133 u32 max_pq_coefs; 134 u32 max_pq_srcs; 135 u32 max_cmd_per_req; 136 u32 max_xor_srcs; 137 u32 max_resp_pool_size; 138 u32 max_cmds_pool_size; 139 /* Mailbox client and Mailbox channels */ 140 struct mbox_client client; 141 struct mbox_chan *mchan; 142 struct device *mbox_dev; 143 /* DMA device and DMA channel */ 144 struct dma_device dma_dev; 145 struct dma_chan dma_chan; 146 /* DMA channel resources */ 147 void *resp_base; 148 dma_addr_t resp_dma_base; 149 void *cmds_base; 150 dma_addr_t cmds_dma_base; 151 spinlock_t reqs_lock; 152 bool reqs_fence; 153 struct list_head reqs_alloc_list; 154 struct list_head reqs_pending_list; 155 struct list_head reqs_active_list; 156 struct list_head reqs_aborted_list; 157 struct list_head reqs_free_list; 158 /* DebugFS directory entries */ 159 struct dentry *root; 160 }; 161 162 /* ====== Command helper routines ===== */ 163 164 static inline u64 __pure sba_cmd_enc(u64 cmd, u32 val, u32 shift, u32 mask) 165 { 166 cmd &= ~((u64)mask << shift); 167 cmd |= ((u64)(val & mask) << shift); 168 return cmd; 169 } 170 171 static inline u32 __pure sba_cmd_load_c_mdata(u32 b0) 172 { 173 return b0 & SBA_C_MDATA_BNUMx_MASK; 174 } 175 176 static inline u32 __pure sba_cmd_write_c_mdata(u32 b0) 177 { 178 return b0 & SBA_C_MDATA_BNUMx_MASK; 179 } 180 181 static inline u32 __pure sba_cmd_xor_c_mdata(u32 b1, u32 b0) 182 { 183 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 184 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)); 185 } 186 187 static inline u32 __pure sba_cmd_pq_c_mdata(u32 d, u32 b1, u32 b0) 188 { 189 return (b0 & SBA_C_MDATA_BNUMx_MASK) | 190 ((b1 & SBA_C_MDATA_BNUMx_MASK) << SBA_C_MDATA_BNUMx_SHIFT(1)) | 191 ((d & SBA_C_MDATA_DNUM_MASK) << SBA_C_MDATA_DNUM_SHIFT); 192 } 193 194 /* ====== General helper routines ===== */ 195 196 static struct sba_request *sba_alloc_request(struct sba_device *sba) 197 { 198 bool found = false; 199 unsigned long flags; 200 struct sba_request *req = NULL; 201 202 spin_lock_irqsave(&sba->reqs_lock, flags); 203 list_for_each_entry(req, &sba->reqs_free_list, node) { 204 if (async_tx_test_ack(&req->tx)) { 205 list_move_tail(&req->node, &sba->reqs_alloc_list); 206 found = true; 207 break; 208 } 209 } 210 spin_unlock_irqrestore(&sba->reqs_lock, flags); 211 212 if (!found) { 213 /* 214 * We have no more free requests so, we peek 215 * mailbox channels hoping few active requests 216 * would have completed which will create more 217 * room for new requests. 218 */ 219 mbox_client_peek_data(sba->mchan); 220 return NULL; 221 } 222 223 req->flags = SBA_REQUEST_STATE_ALLOCED; 224 req->first = req; 225 INIT_LIST_HEAD(&req->next); 226 atomic_set(&req->next_pending_count, 1); 227 228 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 229 async_tx_ack(&req->tx); 230 231 return req; 232 } 233 234 /* Note: Must be called with sba->reqs_lock held */ 235 static void _sba_pending_request(struct sba_device *sba, 236 struct sba_request *req) 237 { 238 lockdep_assert_held(&sba->reqs_lock); 239 req->flags &= ~SBA_REQUEST_STATE_MASK; 240 req->flags |= SBA_REQUEST_STATE_PENDING; 241 list_move_tail(&req->node, &sba->reqs_pending_list); 242 if (list_empty(&sba->reqs_active_list)) 243 sba->reqs_fence = false; 244 } 245 246 /* Note: Must be called with sba->reqs_lock held */ 247 static bool _sba_active_request(struct sba_device *sba, 248 struct sba_request *req) 249 { 250 lockdep_assert_held(&sba->reqs_lock); 251 if (list_empty(&sba->reqs_active_list)) 252 sba->reqs_fence = false; 253 if (sba->reqs_fence) 254 return false; 255 req->flags &= ~SBA_REQUEST_STATE_MASK; 256 req->flags |= SBA_REQUEST_STATE_ACTIVE; 257 list_move_tail(&req->node, &sba->reqs_active_list); 258 if (req->flags & SBA_REQUEST_FENCE) 259 sba->reqs_fence = true; 260 return true; 261 } 262 263 /* Note: Must be called with sba->reqs_lock held */ 264 static void _sba_abort_request(struct sba_device *sba, 265 struct sba_request *req) 266 { 267 lockdep_assert_held(&sba->reqs_lock); 268 req->flags &= ~SBA_REQUEST_STATE_MASK; 269 req->flags |= SBA_REQUEST_STATE_ABORTED; 270 list_move_tail(&req->node, &sba->reqs_aborted_list); 271 if (list_empty(&sba->reqs_active_list)) 272 sba->reqs_fence = false; 273 } 274 275 /* Note: Must be called with sba->reqs_lock held */ 276 static void _sba_free_request(struct sba_device *sba, 277 struct sba_request *req) 278 { 279 lockdep_assert_held(&sba->reqs_lock); 280 req->flags &= ~SBA_REQUEST_STATE_MASK; 281 req->flags |= SBA_REQUEST_STATE_FREE; 282 list_move_tail(&req->node, &sba->reqs_free_list); 283 if (list_empty(&sba->reqs_active_list)) 284 sba->reqs_fence = false; 285 } 286 287 static void sba_free_chained_requests(struct sba_request *req) 288 { 289 unsigned long flags; 290 struct sba_request *nreq; 291 struct sba_device *sba = req->sba; 292 293 spin_lock_irqsave(&sba->reqs_lock, flags); 294 295 _sba_free_request(sba, req); 296 list_for_each_entry(nreq, &req->next, next) 297 _sba_free_request(sba, nreq); 298 299 spin_unlock_irqrestore(&sba->reqs_lock, flags); 300 } 301 302 static void sba_chain_request(struct sba_request *first, 303 struct sba_request *req) 304 { 305 unsigned long flags; 306 struct sba_device *sba = req->sba; 307 308 spin_lock_irqsave(&sba->reqs_lock, flags); 309 310 list_add_tail(&req->next, &first->next); 311 req->first = first; 312 atomic_inc(&first->next_pending_count); 313 314 spin_unlock_irqrestore(&sba->reqs_lock, flags); 315 } 316 317 static void sba_cleanup_nonpending_requests(struct sba_device *sba) 318 { 319 unsigned long flags; 320 struct sba_request *req, *req1; 321 322 spin_lock_irqsave(&sba->reqs_lock, flags); 323 324 /* Freeup all alloced request */ 325 list_for_each_entry_safe(req, req1, &sba->reqs_alloc_list, node) 326 _sba_free_request(sba, req); 327 328 /* Set all active requests as aborted */ 329 list_for_each_entry_safe(req, req1, &sba->reqs_active_list, node) 330 _sba_abort_request(sba, req); 331 332 /* 333 * Note: We expect that aborted request will be eventually 334 * freed by sba_receive_message() 335 */ 336 337 spin_unlock_irqrestore(&sba->reqs_lock, flags); 338 } 339 340 static void sba_cleanup_pending_requests(struct sba_device *sba) 341 { 342 unsigned long flags; 343 struct sba_request *req, *req1; 344 345 spin_lock_irqsave(&sba->reqs_lock, flags); 346 347 /* Freeup all pending request */ 348 list_for_each_entry_safe(req, req1, &sba->reqs_pending_list, node) 349 _sba_free_request(sba, req); 350 351 spin_unlock_irqrestore(&sba->reqs_lock, flags); 352 } 353 354 static int sba_send_mbox_request(struct sba_device *sba, 355 struct sba_request *req) 356 { 357 int ret = 0; 358 359 /* Send message for the request */ 360 req->msg.error = 0; 361 ret = mbox_send_message(sba->mchan, &req->msg); 362 if (ret < 0) { 363 dev_err(sba->dev, "send message failed with error %d", ret); 364 return ret; 365 } 366 367 /* Check error returned by mailbox controller */ 368 ret = req->msg.error; 369 if (ret < 0) { 370 dev_err(sba->dev, "message error %d", ret); 371 } 372 373 /* Signal txdone for mailbox channel */ 374 mbox_client_txdone(sba->mchan, ret); 375 376 return ret; 377 } 378 379 /* Note: Must be called with sba->reqs_lock held */ 380 static void _sba_process_pending_requests(struct sba_device *sba) 381 { 382 int ret; 383 u32 count; 384 struct sba_request *req; 385 386 /* Process few pending requests */ 387 count = SBA_MAX_MSG_SEND_PER_MBOX_CHANNEL; 388 while (!list_empty(&sba->reqs_pending_list) && count) { 389 /* Get the first pending request */ 390 req = list_first_entry(&sba->reqs_pending_list, 391 struct sba_request, node); 392 393 /* Try to make request active */ 394 if (!_sba_active_request(sba, req)) 395 break; 396 397 /* Send request to mailbox channel */ 398 ret = sba_send_mbox_request(sba, req); 399 if (ret < 0) { 400 _sba_pending_request(sba, req); 401 break; 402 } 403 404 count--; 405 } 406 } 407 408 static void sba_process_received_request(struct sba_device *sba, 409 struct sba_request *req) 410 { 411 unsigned long flags; 412 struct dma_async_tx_descriptor *tx; 413 struct sba_request *nreq, *first = req->first; 414 415 /* Process only after all chained requests are received */ 416 if (!atomic_dec_return(&first->next_pending_count)) { 417 tx = &first->tx; 418 419 WARN_ON(tx->cookie < 0); 420 if (tx->cookie > 0) { 421 spin_lock_irqsave(&sba->reqs_lock, flags); 422 dma_cookie_complete(tx); 423 spin_unlock_irqrestore(&sba->reqs_lock, flags); 424 dmaengine_desc_get_callback_invoke(tx, NULL); 425 dma_descriptor_unmap(tx); 426 tx->callback = NULL; 427 tx->callback_result = NULL; 428 } 429 430 dma_run_dependencies(tx); 431 432 spin_lock_irqsave(&sba->reqs_lock, flags); 433 434 /* Free all requests chained to first request */ 435 list_for_each_entry(nreq, &first->next, next) 436 _sba_free_request(sba, nreq); 437 INIT_LIST_HEAD(&first->next); 438 439 /* Free the first request */ 440 _sba_free_request(sba, first); 441 442 /* Process pending requests */ 443 _sba_process_pending_requests(sba); 444 445 spin_unlock_irqrestore(&sba->reqs_lock, flags); 446 } 447 } 448 449 static void sba_write_stats_in_seqfile(struct sba_device *sba, 450 struct seq_file *file) 451 { 452 unsigned long flags; 453 struct sba_request *req; 454 u32 free_count = 0, alloced_count = 0; 455 u32 pending_count = 0, active_count = 0, aborted_count = 0; 456 457 spin_lock_irqsave(&sba->reqs_lock, flags); 458 459 list_for_each_entry(req, &sba->reqs_free_list, node) 460 if (async_tx_test_ack(&req->tx)) 461 free_count++; 462 463 list_for_each_entry(req, &sba->reqs_alloc_list, node) 464 alloced_count++; 465 466 list_for_each_entry(req, &sba->reqs_pending_list, node) 467 pending_count++; 468 469 list_for_each_entry(req, &sba->reqs_active_list, node) 470 active_count++; 471 472 list_for_each_entry(req, &sba->reqs_aborted_list, node) 473 aborted_count++; 474 475 spin_unlock_irqrestore(&sba->reqs_lock, flags); 476 477 seq_printf(file, "maximum requests = %d\n", sba->max_req); 478 seq_printf(file, "free requests = %d\n", free_count); 479 seq_printf(file, "alloced requests = %d\n", alloced_count); 480 seq_printf(file, "pending requests = %d\n", pending_count); 481 seq_printf(file, "active requests = %d\n", active_count); 482 seq_printf(file, "aborted requests = %d\n", aborted_count); 483 } 484 485 /* ====== DMAENGINE callbacks ===== */ 486 487 static void sba_free_chan_resources(struct dma_chan *dchan) 488 { 489 /* 490 * Channel resources are pre-alloced so we just free-up 491 * whatever we can so that we can re-use pre-alloced 492 * channel resources next time. 493 */ 494 sba_cleanup_nonpending_requests(to_sba_device(dchan)); 495 } 496 497 static int sba_device_terminate_all(struct dma_chan *dchan) 498 { 499 /* Cleanup all pending requests */ 500 sba_cleanup_pending_requests(to_sba_device(dchan)); 501 502 return 0; 503 } 504 505 static void sba_issue_pending(struct dma_chan *dchan) 506 { 507 unsigned long flags; 508 struct sba_device *sba = to_sba_device(dchan); 509 510 /* Process pending requests */ 511 spin_lock_irqsave(&sba->reqs_lock, flags); 512 _sba_process_pending_requests(sba); 513 spin_unlock_irqrestore(&sba->reqs_lock, flags); 514 } 515 516 static dma_cookie_t sba_tx_submit(struct dma_async_tx_descriptor *tx) 517 { 518 unsigned long flags; 519 dma_cookie_t cookie; 520 struct sba_device *sba; 521 struct sba_request *req, *nreq; 522 523 if (unlikely(!tx)) 524 return -EINVAL; 525 526 sba = to_sba_device(tx->chan); 527 req = to_sba_request(tx); 528 529 /* Assign cookie and mark all chained requests pending */ 530 spin_lock_irqsave(&sba->reqs_lock, flags); 531 cookie = dma_cookie_assign(tx); 532 _sba_pending_request(sba, req); 533 list_for_each_entry(nreq, &req->next, next) 534 _sba_pending_request(sba, nreq); 535 spin_unlock_irqrestore(&sba->reqs_lock, flags); 536 537 return cookie; 538 } 539 540 static enum dma_status sba_tx_status(struct dma_chan *dchan, 541 dma_cookie_t cookie, 542 struct dma_tx_state *txstate) 543 { 544 enum dma_status ret; 545 struct sba_device *sba = to_sba_device(dchan); 546 547 ret = dma_cookie_status(dchan, cookie, txstate); 548 if (ret == DMA_COMPLETE) 549 return ret; 550 551 mbox_client_peek_data(sba->mchan); 552 553 return dma_cookie_status(dchan, cookie, txstate); 554 } 555 556 static void sba_fillup_interrupt_msg(struct sba_request *req, 557 struct brcm_sba_command *cmds, 558 struct brcm_message *msg) 559 { 560 u64 cmd; 561 u32 c_mdata; 562 dma_addr_t resp_dma = req->tx.phys; 563 struct brcm_sba_command *cmdsp = cmds; 564 565 /* Type-B command to load dummy data into buf0 */ 566 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 567 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 568 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 569 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 570 c_mdata = sba_cmd_load_c_mdata(0); 571 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 572 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 573 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 574 SBA_CMD_SHIFT, SBA_CMD_MASK); 575 cmdsp->cmd = cmd; 576 *cmdsp->cmd_dma = cpu_to_le64(cmd); 577 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 578 cmdsp->data = resp_dma; 579 cmdsp->data_len = req->sba->hw_resp_size; 580 cmdsp++; 581 582 /* Type-A command to write buf0 to dummy location */ 583 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 584 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 585 cmd = sba_cmd_enc(cmd, req->sba->hw_resp_size, 586 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 587 cmd = sba_cmd_enc(cmd, 0x1, 588 SBA_RESP_SHIFT, SBA_RESP_MASK); 589 c_mdata = sba_cmd_write_c_mdata(0); 590 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 591 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 592 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 593 SBA_CMD_SHIFT, SBA_CMD_MASK); 594 cmdsp->cmd = cmd; 595 *cmdsp->cmd_dma = cpu_to_le64(cmd); 596 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 597 if (req->sba->hw_resp_size) { 598 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 599 cmdsp->resp = resp_dma; 600 cmdsp->resp_len = req->sba->hw_resp_size; 601 } 602 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 603 cmdsp->data = resp_dma; 604 cmdsp->data_len = req->sba->hw_resp_size; 605 cmdsp++; 606 607 /* Fillup brcm_message */ 608 msg->type = BRCM_MESSAGE_SBA; 609 msg->sba.cmds = cmds; 610 msg->sba.cmds_count = cmdsp - cmds; 611 msg->ctx = req; 612 msg->error = 0; 613 } 614 615 static struct dma_async_tx_descriptor * 616 sba_prep_dma_interrupt(struct dma_chan *dchan, unsigned long flags) 617 { 618 struct sba_request *req = NULL; 619 struct sba_device *sba = to_sba_device(dchan); 620 621 /* Alloc new request */ 622 req = sba_alloc_request(sba); 623 if (!req) 624 return NULL; 625 626 /* 627 * Force fence so that no requests are submitted 628 * until DMA callback for this request is invoked. 629 */ 630 req->flags |= SBA_REQUEST_FENCE; 631 632 /* Fillup request message */ 633 sba_fillup_interrupt_msg(req, req->cmds, &req->msg); 634 635 /* Init async_tx descriptor */ 636 req->tx.flags = flags; 637 req->tx.cookie = -EBUSY; 638 639 return &req->tx; 640 } 641 642 static void sba_fillup_memcpy_msg(struct sba_request *req, 643 struct brcm_sba_command *cmds, 644 struct brcm_message *msg, 645 dma_addr_t msg_offset, size_t msg_len, 646 dma_addr_t dst, dma_addr_t src) 647 { 648 u64 cmd; 649 u32 c_mdata; 650 dma_addr_t resp_dma = req->tx.phys; 651 struct brcm_sba_command *cmdsp = cmds; 652 653 /* Type-B command to load data into buf0 */ 654 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 655 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 656 cmd = sba_cmd_enc(cmd, msg_len, 657 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 658 c_mdata = sba_cmd_load_c_mdata(0); 659 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 660 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 661 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 662 SBA_CMD_SHIFT, SBA_CMD_MASK); 663 cmdsp->cmd = cmd; 664 *cmdsp->cmd_dma = cpu_to_le64(cmd); 665 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 666 cmdsp->data = src + msg_offset; 667 cmdsp->data_len = msg_len; 668 cmdsp++; 669 670 /* Type-A command to write buf0 */ 671 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 672 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 673 cmd = sba_cmd_enc(cmd, msg_len, 674 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 675 cmd = sba_cmd_enc(cmd, 0x1, 676 SBA_RESP_SHIFT, SBA_RESP_MASK); 677 c_mdata = sba_cmd_write_c_mdata(0); 678 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 679 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 680 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 681 SBA_CMD_SHIFT, SBA_CMD_MASK); 682 cmdsp->cmd = cmd; 683 *cmdsp->cmd_dma = cpu_to_le64(cmd); 684 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 685 if (req->sba->hw_resp_size) { 686 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 687 cmdsp->resp = resp_dma; 688 cmdsp->resp_len = req->sba->hw_resp_size; 689 } 690 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 691 cmdsp->data = dst + msg_offset; 692 cmdsp->data_len = msg_len; 693 cmdsp++; 694 695 /* Fillup brcm_message */ 696 msg->type = BRCM_MESSAGE_SBA; 697 msg->sba.cmds = cmds; 698 msg->sba.cmds_count = cmdsp - cmds; 699 msg->ctx = req; 700 msg->error = 0; 701 } 702 703 static struct sba_request * 704 sba_prep_dma_memcpy_req(struct sba_device *sba, 705 dma_addr_t off, dma_addr_t dst, dma_addr_t src, 706 size_t len, unsigned long flags) 707 { 708 struct sba_request *req = NULL; 709 710 /* Alloc new request */ 711 req = sba_alloc_request(sba); 712 if (!req) 713 return NULL; 714 if (flags & DMA_PREP_FENCE) 715 req->flags |= SBA_REQUEST_FENCE; 716 717 /* Fillup request message */ 718 sba_fillup_memcpy_msg(req, req->cmds, &req->msg, 719 off, len, dst, src); 720 721 /* Init async_tx descriptor */ 722 req->tx.flags = flags; 723 req->tx.cookie = -EBUSY; 724 725 return req; 726 } 727 728 static struct dma_async_tx_descriptor * 729 sba_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t src, 730 size_t len, unsigned long flags) 731 { 732 size_t req_len; 733 dma_addr_t off = 0; 734 struct sba_device *sba = to_sba_device(dchan); 735 struct sba_request *first = NULL, *req; 736 737 /* Create chained requests where each request is upto hw_buf_size */ 738 while (len) { 739 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 740 741 req = sba_prep_dma_memcpy_req(sba, off, dst, src, 742 req_len, flags); 743 if (!req) { 744 if (first) 745 sba_free_chained_requests(first); 746 return NULL; 747 } 748 749 if (first) 750 sba_chain_request(first, req); 751 else 752 first = req; 753 754 off += req_len; 755 len -= req_len; 756 } 757 758 return (first) ? &first->tx : NULL; 759 } 760 761 static void sba_fillup_xor_msg(struct sba_request *req, 762 struct brcm_sba_command *cmds, 763 struct brcm_message *msg, 764 dma_addr_t msg_offset, size_t msg_len, 765 dma_addr_t dst, dma_addr_t *src, u32 src_cnt) 766 { 767 u64 cmd; 768 u32 c_mdata; 769 unsigned int i; 770 dma_addr_t resp_dma = req->tx.phys; 771 struct brcm_sba_command *cmdsp = cmds; 772 773 /* Type-B command to load data into buf0 */ 774 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 775 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 776 cmd = sba_cmd_enc(cmd, msg_len, 777 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 778 c_mdata = sba_cmd_load_c_mdata(0); 779 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 780 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 781 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 782 SBA_CMD_SHIFT, SBA_CMD_MASK); 783 cmdsp->cmd = cmd; 784 *cmdsp->cmd_dma = cpu_to_le64(cmd); 785 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 786 cmdsp->data = src[0] + msg_offset; 787 cmdsp->data_len = msg_len; 788 cmdsp++; 789 790 /* Type-B commands to xor data with buf0 and put it back in buf0 */ 791 for (i = 1; i < src_cnt; i++) { 792 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 793 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 794 cmd = sba_cmd_enc(cmd, msg_len, 795 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 796 c_mdata = sba_cmd_xor_c_mdata(0, 0); 797 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 798 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 799 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 800 SBA_CMD_SHIFT, SBA_CMD_MASK); 801 cmdsp->cmd = cmd; 802 *cmdsp->cmd_dma = cpu_to_le64(cmd); 803 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 804 cmdsp->data = src[i] + msg_offset; 805 cmdsp->data_len = msg_len; 806 cmdsp++; 807 } 808 809 /* Type-A command to write buf0 */ 810 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 811 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 812 cmd = sba_cmd_enc(cmd, msg_len, 813 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 814 cmd = sba_cmd_enc(cmd, 0x1, 815 SBA_RESP_SHIFT, SBA_RESP_MASK); 816 c_mdata = sba_cmd_write_c_mdata(0); 817 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 818 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 819 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 820 SBA_CMD_SHIFT, SBA_CMD_MASK); 821 cmdsp->cmd = cmd; 822 *cmdsp->cmd_dma = cpu_to_le64(cmd); 823 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 824 if (req->sba->hw_resp_size) { 825 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 826 cmdsp->resp = resp_dma; 827 cmdsp->resp_len = req->sba->hw_resp_size; 828 } 829 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 830 cmdsp->data = dst + msg_offset; 831 cmdsp->data_len = msg_len; 832 cmdsp++; 833 834 /* Fillup brcm_message */ 835 msg->type = BRCM_MESSAGE_SBA; 836 msg->sba.cmds = cmds; 837 msg->sba.cmds_count = cmdsp - cmds; 838 msg->ctx = req; 839 msg->error = 0; 840 } 841 842 static struct sba_request * 843 sba_prep_dma_xor_req(struct sba_device *sba, 844 dma_addr_t off, dma_addr_t dst, dma_addr_t *src, 845 u32 src_cnt, size_t len, unsigned long flags) 846 { 847 struct sba_request *req = NULL; 848 849 /* Alloc new request */ 850 req = sba_alloc_request(sba); 851 if (!req) 852 return NULL; 853 if (flags & DMA_PREP_FENCE) 854 req->flags |= SBA_REQUEST_FENCE; 855 856 /* Fillup request message */ 857 sba_fillup_xor_msg(req, req->cmds, &req->msg, 858 off, len, dst, src, src_cnt); 859 860 /* Init async_tx descriptor */ 861 req->tx.flags = flags; 862 req->tx.cookie = -EBUSY; 863 864 return req; 865 } 866 867 static struct dma_async_tx_descriptor * 868 sba_prep_dma_xor(struct dma_chan *dchan, dma_addr_t dst, dma_addr_t *src, 869 u32 src_cnt, size_t len, unsigned long flags) 870 { 871 size_t req_len; 872 dma_addr_t off = 0; 873 struct sba_device *sba = to_sba_device(dchan); 874 struct sba_request *first = NULL, *req; 875 876 /* Sanity checks */ 877 if (unlikely(src_cnt > sba->max_xor_srcs)) 878 return NULL; 879 880 /* Create chained requests where each request is upto hw_buf_size */ 881 while (len) { 882 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 883 884 req = sba_prep_dma_xor_req(sba, off, dst, src, src_cnt, 885 req_len, flags); 886 if (!req) { 887 if (first) 888 sba_free_chained_requests(first); 889 return NULL; 890 } 891 892 if (first) 893 sba_chain_request(first, req); 894 else 895 first = req; 896 897 off += req_len; 898 len -= req_len; 899 } 900 901 return (first) ? &first->tx : NULL; 902 } 903 904 static void sba_fillup_pq_msg(struct sba_request *req, 905 bool pq_continue, 906 struct brcm_sba_command *cmds, 907 struct brcm_message *msg, 908 dma_addr_t msg_offset, size_t msg_len, 909 dma_addr_t *dst_p, dma_addr_t *dst_q, 910 const u8 *scf, dma_addr_t *src, u32 src_cnt) 911 { 912 u64 cmd; 913 u32 c_mdata; 914 unsigned int i; 915 dma_addr_t resp_dma = req->tx.phys; 916 struct brcm_sba_command *cmdsp = cmds; 917 918 if (pq_continue) { 919 /* Type-B command to load old P into buf0 */ 920 if (dst_p) { 921 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 922 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 923 cmd = sba_cmd_enc(cmd, msg_len, 924 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 925 c_mdata = sba_cmd_load_c_mdata(0); 926 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 927 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 928 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 929 SBA_CMD_SHIFT, SBA_CMD_MASK); 930 cmdsp->cmd = cmd; 931 *cmdsp->cmd_dma = cpu_to_le64(cmd); 932 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 933 cmdsp->data = *dst_p + msg_offset; 934 cmdsp->data_len = msg_len; 935 cmdsp++; 936 } 937 938 /* Type-B command to load old Q into buf1 */ 939 if (dst_q) { 940 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 941 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 942 cmd = sba_cmd_enc(cmd, msg_len, 943 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 944 c_mdata = sba_cmd_load_c_mdata(1); 945 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 946 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 947 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 948 SBA_CMD_SHIFT, SBA_CMD_MASK); 949 cmdsp->cmd = cmd; 950 *cmdsp->cmd_dma = cpu_to_le64(cmd); 951 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 952 cmdsp->data = *dst_q + msg_offset; 953 cmdsp->data_len = msg_len; 954 cmdsp++; 955 } 956 } else { 957 /* Type-A command to zero all buffers */ 958 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 959 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 960 cmd = sba_cmd_enc(cmd, msg_len, 961 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 962 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 963 SBA_CMD_SHIFT, SBA_CMD_MASK); 964 cmdsp->cmd = cmd; 965 *cmdsp->cmd_dma = cpu_to_le64(cmd); 966 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 967 cmdsp++; 968 } 969 970 /* Type-B commands for generate P onto buf0 and Q onto buf1 */ 971 for (i = 0; i < src_cnt; i++) { 972 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 973 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 974 cmd = sba_cmd_enc(cmd, msg_len, 975 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 976 c_mdata = sba_cmd_pq_c_mdata(raid6_gflog[scf[i]], 1, 0); 977 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 978 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 979 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 980 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 981 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS_XOR, 982 SBA_CMD_SHIFT, SBA_CMD_MASK); 983 cmdsp->cmd = cmd; 984 *cmdsp->cmd_dma = cpu_to_le64(cmd); 985 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 986 cmdsp->data = src[i] + msg_offset; 987 cmdsp->data_len = msg_len; 988 cmdsp++; 989 } 990 991 /* Type-A command to write buf0 */ 992 if (dst_p) { 993 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 994 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 995 cmd = sba_cmd_enc(cmd, msg_len, 996 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 997 cmd = sba_cmd_enc(cmd, 0x1, 998 SBA_RESP_SHIFT, SBA_RESP_MASK); 999 c_mdata = sba_cmd_write_c_mdata(0); 1000 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1001 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1002 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1003 SBA_CMD_SHIFT, SBA_CMD_MASK); 1004 cmdsp->cmd = cmd; 1005 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1006 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1007 if (req->sba->hw_resp_size) { 1008 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1009 cmdsp->resp = resp_dma; 1010 cmdsp->resp_len = req->sba->hw_resp_size; 1011 } 1012 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1013 cmdsp->data = *dst_p + msg_offset; 1014 cmdsp->data_len = msg_len; 1015 cmdsp++; 1016 } 1017 1018 /* Type-A command to write buf1 */ 1019 if (dst_q) { 1020 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1021 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1022 cmd = sba_cmd_enc(cmd, msg_len, 1023 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1024 cmd = sba_cmd_enc(cmd, 0x1, 1025 SBA_RESP_SHIFT, SBA_RESP_MASK); 1026 c_mdata = sba_cmd_write_c_mdata(1); 1027 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1028 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1029 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1030 SBA_CMD_SHIFT, SBA_CMD_MASK); 1031 cmdsp->cmd = cmd; 1032 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1033 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1034 if (req->sba->hw_resp_size) { 1035 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1036 cmdsp->resp = resp_dma; 1037 cmdsp->resp_len = req->sba->hw_resp_size; 1038 } 1039 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1040 cmdsp->data = *dst_q + msg_offset; 1041 cmdsp->data_len = msg_len; 1042 cmdsp++; 1043 } 1044 1045 /* Fillup brcm_message */ 1046 msg->type = BRCM_MESSAGE_SBA; 1047 msg->sba.cmds = cmds; 1048 msg->sba.cmds_count = cmdsp - cmds; 1049 msg->ctx = req; 1050 msg->error = 0; 1051 } 1052 1053 static struct sba_request * 1054 sba_prep_dma_pq_req(struct sba_device *sba, dma_addr_t off, 1055 dma_addr_t *dst_p, dma_addr_t *dst_q, dma_addr_t *src, 1056 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1057 { 1058 struct sba_request *req = NULL; 1059 1060 /* Alloc new request */ 1061 req = sba_alloc_request(sba); 1062 if (!req) 1063 return NULL; 1064 if (flags & DMA_PREP_FENCE) 1065 req->flags |= SBA_REQUEST_FENCE; 1066 1067 /* Fillup request messages */ 1068 sba_fillup_pq_msg(req, dmaf_continue(flags), 1069 req->cmds, &req->msg, 1070 off, len, dst_p, dst_q, scf, src, src_cnt); 1071 1072 /* Init async_tx descriptor */ 1073 req->tx.flags = flags; 1074 req->tx.cookie = -EBUSY; 1075 1076 return req; 1077 } 1078 1079 static void sba_fillup_pq_single_msg(struct sba_request *req, 1080 bool pq_continue, 1081 struct brcm_sba_command *cmds, 1082 struct brcm_message *msg, 1083 dma_addr_t msg_offset, size_t msg_len, 1084 dma_addr_t *dst_p, dma_addr_t *dst_q, 1085 dma_addr_t src, u8 scf) 1086 { 1087 u64 cmd; 1088 u32 c_mdata; 1089 u8 pos, dpos = raid6_gflog[scf]; 1090 dma_addr_t resp_dma = req->tx.phys; 1091 struct brcm_sba_command *cmdsp = cmds; 1092 1093 if (!dst_p) 1094 goto skip_p; 1095 1096 if (pq_continue) { 1097 /* Type-B command to load old P into buf0 */ 1098 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1099 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1100 cmd = sba_cmd_enc(cmd, msg_len, 1101 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1102 c_mdata = sba_cmd_load_c_mdata(0); 1103 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1104 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1105 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1106 SBA_CMD_SHIFT, SBA_CMD_MASK); 1107 cmdsp->cmd = cmd; 1108 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1109 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1110 cmdsp->data = *dst_p + msg_offset; 1111 cmdsp->data_len = msg_len; 1112 cmdsp++; 1113 1114 /* 1115 * Type-B commands to xor data with buf0 and put it 1116 * back in buf0 1117 */ 1118 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1119 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1120 cmd = sba_cmd_enc(cmd, msg_len, 1121 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1122 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1123 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1124 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1125 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1126 SBA_CMD_SHIFT, SBA_CMD_MASK); 1127 cmdsp->cmd = cmd; 1128 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1129 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1130 cmdsp->data = src + msg_offset; 1131 cmdsp->data_len = msg_len; 1132 cmdsp++; 1133 } else { 1134 /* Type-B command to load old P into buf0 */ 1135 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1136 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1137 cmd = sba_cmd_enc(cmd, msg_len, 1138 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1139 c_mdata = sba_cmd_load_c_mdata(0); 1140 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1141 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1142 cmd = sba_cmd_enc(cmd, SBA_CMD_LOAD_BUFFER, 1143 SBA_CMD_SHIFT, SBA_CMD_MASK); 1144 cmdsp->cmd = cmd; 1145 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1146 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1147 cmdsp->data = src + msg_offset; 1148 cmdsp->data_len = msg_len; 1149 cmdsp++; 1150 } 1151 1152 /* Type-A command to write buf0 */ 1153 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1154 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1155 cmd = sba_cmd_enc(cmd, msg_len, 1156 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1157 cmd = sba_cmd_enc(cmd, 0x1, 1158 SBA_RESP_SHIFT, SBA_RESP_MASK); 1159 c_mdata = sba_cmd_write_c_mdata(0); 1160 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1161 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1162 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1163 SBA_CMD_SHIFT, SBA_CMD_MASK); 1164 cmdsp->cmd = cmd; 1165 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1166 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1167 if (req->sba->hw_resp_size) { 1168 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1169 cmdsp->resp = resp_dma; 1170 cmdsp->resp_len = req->sba->hw_resp_size; 1171 } 1172 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1173 cmdsp->data = *dst_p + msg_offset; 1174 cmdsp->data_len = msg_len; 1175 cmdsp++; 1176 1177 skip_p: 1178 if (!dst_q) 1179 goto skip_q; 1180 1181 /* Type-A command to zero all buffers */ 1182 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1183 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1184 cmd = sba_cmd_enc(cmd, msg_len, 1185 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1186 cmd = sba_cmd_enc(cmd, SBA_CMD_ZERO_ALL_BUFFERS, 1187 SBA_CMD_SHIFT, SBA_CMD_MASK); 1188 cmdsp->cmd = cmd; 1189 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1190 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1191 cmdsp++; 1192 1193 if (dpos == 255) 1194 goto skip_q_computation; 1195 pos = (dpos < req->sba->max_pq_coefs) ? 1196 dpos : (req->sba->max_pq_coefs - 1); 1197 1198 /* 1199 * Type-B command to generate initial Q from data 1200 * and store output into buf0 1201 */ 1202 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1203 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1204 cmd = sba_cmd_enc(cmd, msg_len, 1205 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1206 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 0); 1207 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1208 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1209 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1210 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1211 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1212 SBA_CMD_SHIFT, SBA_CMD_MASK); 1213 cmdsp->cmd = cmd; 1214 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1215 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1216 cmdsp->data = src + msg_offset; 1217 cmdsp->data_len = msg_len; 1218 cmdsp++; 1219 1220 dpos -= pos; 1221 1222 /* Multiple Type-A command to generate final Q */ 1223 while (dpos) { 1224 pos = (dpos < req->sba->max_pq_coefs) ? 1225 dpos : (req->sba->max_pq_coefs - 1); 1226 1227 /* 1228 * Type-A command to generate Q with buf0 and 1229 * buf1 store result in buf0 1230 */ 1231 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1232 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1233 cmd = sba_cmd_enc(cmd, msg_len, 1234 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1235 c_mdata = sba_cmd_pq_c_mdata(pos, 0, 1); 1236 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1237 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1238 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_MS(c_mdata), 1239 SBA_C_MDATA_MS_SHIFT, SBA_C_MDATA_MS_MASK); 1240 cmd = sba_cmd_enc(cmd, SBA_CMD_GALOIS, 1241 SBA_CMD_SHIFT, SBA_CMD_MASK); 1242 cmdsp->cmd = cmd; 1243 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1244 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1245 cmdsp++; 1246 1247 dpos -= pos; 1248 } 1249 1250 skip_q_computation: 1251 if (pq_continue) { 1252 /* 1253 * Type-B command to XOR previous output with 1254 * buf0 and write it into buf0 1255 */ 1256 cmd = sba_cmd_enc(0x0, SBA_TYPE_B, 1257 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1258 cmd = sba_cmd_enc(cmd, msg_len, 1259 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1260 c_mdata = sba_cmd_xor_c_mdata(0, 0); 1261 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1262 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1263 cmd = sba_cmd_enc(cmd, SBA_CMD_XOR, 1264 SBA_CMD_SHIFT, SBA_CMD_MASK); 1265 cmdsp->cmd = cmd; 1266 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1267 cmdsp->flags = BRCM_SBA_CMD_TYPE_B; 1268 cmdsp->data = *dst_q + msg_offset; 1269 cmdsp->data_len = msg_len; 1270 cmdsp++; 1271 } 1272 1273 /* Type-A command to write buf0 */ 1274 cmd = sba_cmd_enc(0x0, SBA_TYPE_A, 1275 SBA_TYPE_SHIFT, SBA_TYPE_MASK); 1276 cmd = sba_cmd_enc(cmd, msg_len, 1277 SBA_USER_DEF_SHIFT, SBA_USER_DEF_MASK); 1278 cmd = sba_cmd_enc(cmd, 0x1, 1279 SBA_RESP_SHIFT, SBA_RESP_MASK); 1280 c_mdata = sba_cmd_write_c_mdata(0); 1281 cmd = sba_cmd_enc(cmd, SBA_C_MDATA_LS(c_mdata), 1282 SBA_C_MDATA_SHIFT, SBA_C_MDATA_MASK); 1283 cmd = sba_cmd_enc(cmd, SBA_CMD_WRITE_BUFFER, 1284 SBA_CMD_SHIFT, SBA_CMD_MASK); 1285 cmdsp->cmd = cmd; 1286 *cmdsp->cmd_dma = cpu_to_le64(cmd); 1287 cmdsp->flags = BRCM_SBA_CMD_TYPE_A; 1288 if (req->sba->hw_resp_size) { 1289 cmdsp->flags |= BRCM_SBA_CMD_HAS_RESP; 1290 cmdsp->resp = resp_dma; 1291 cmdsp->resp_len = req->sba->hw_resp_size; 1292 } 1293 cmdsp->flags |= BRCM_SBA_CMD_HAS_OUTPUT; 1294 cmdsp->data = *dst_q + msg_offset; 1295 cmdsp->data_len = msg_len; 1296 cmdsp++; 1297 1298 skip_q: 1299 /* Fillup brcm_message */ 1300 msg->type = BRCM_MESSAGE_SBA; 1301 msg->sba.cmds = cmds; 1302 msg->sba.cmds_count = cmdsp - cmds; 1303 msg->ctx = req; 1304 msg->error = 0; 1305 } 1306 1307 static struct sba_request * 1308 sba_prep_dma_pq_single_req(struct sba_device *sba, dma_addr_t off, 1309 dma_addr_t *dst_p, dma_addr_t *dst_q, 1310 dma_addr_t src, u8 scf, size_t len, 1311 unsigned long flags) 1312 { 1313 struct sba_request *req = NULL; 1314 1315 /* Alloc new request */ 1316 req = sba_alloc_request(sba); 1317 if (!req) 1318 return NULL; 1319 if (flags & DMA_PREP_FENCE) 1320 req->flags |= SBA_REQUEST_FENCE; 1321 1322 /* Fillup request messages */ 1323 sba_fillup_pq_single_msg(req, dmaf_continue(flags), 1324 req->cmds, &req->msg, off, len, 1325 dst_p, dst_q, src, scf); 1326 1327 /* Init async_tx descriptor */ 1328 req->tx.flags = flags; 1329 req->tx.cookie = -EBUSY; 1330 1331 return req; 1332 } 1333 1334 static struct dma_async_tx_descriptor * 1335 sba_prep_dma_pq(struct dma_chan *dchan, dma_addr_t *dst, dma_addr_t *src, 1336 u32 src_cnt, const u8 *scf, size_t len, unsigned long flags) 1337 { 1338 u32 i, dst_q_index; 1339 size_t req_len; 1340 bool slow = false; 1341 dma_addr_t off = 0; 1342 dma_addr_t *dst_p = NULL, *dst_q = NULL; 1343 struct sba_device *sba = to_sba_device(dchan); 1344 struct sba_request *first = NULL, *req; 1345 1346 /* Sanity checks */ 1347 if (unlikely(src_cnt > sba->max_pq_srcs)) 1348 return NULL; 1349 for (i = 0; i < src_cnt; i++) 1350 if (sba->max_pq_coefs <= raid6_gflog[scf[i]]) 1351 slow = true; 1352 1353 /* Figure-out P and Q destination addresses */ 1354 if (!(flags & DMA_PREP_PQ_DISABLE_P)) 1355 dst_p = &dst[0]; 1356 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 1357 dst_q = &dst[1]; 1358 1359 /* Create chained requests where each request is upto hw_buf_size */ 1360 while (len) { 1361 req_len = (len < sba->hw_buf_size) ? len : sba->hw_buf_size; 1362 1363 if (slow) { 1364 dst_q_index = src_cnt; 1365 1366 if (dst_q) { 1367 for (i = 0; i < src_cnt; i++) { 1368 if (*dst_q == src[i]) { 1369 dst_q_index = i; 1370 break; 1371 } 1372 } 1373 } 1374 1375 if (dst_q_index < src_cnt) { 1376 i = dst_q_index; 1377 req = sba_prep_dma_pq_single_req(sba, 1378 off, dst_p, dst_q, src[i], scf[i], 1379 req_len, flags | DMA_PREP_FENCE); 1380 if (!req) 1381 goto fail; 1382 1383 if (first) 1384 sba_chain_request(first, req); 1385 else 1386 first = req; 1387 1388 flags |= DMA_PREP_CONTINUE; 1389 } 1390 1391 for (i = 0; i < src_cnt; i++) { 1392 if (dst_q_index == i) 1393 continue; 1394 1395 req = sba_prep_dma_pq_single_req(sba, 1396 off, dst_p, dst_q, src[i], scf[i], 1397 req_len, flags | DMA_PREP_FENCE); 1398 if (!req) 1399 goto fail; 1400 1401 if (first) 1402 sba_chain_request(first, req); 1403 else 1404 first = req; 1405 1406 flags |= DMA_PREP_CONTINUE; 1407 } 1408 } else { 1409 req = sba_prep_dma_pq_req(sba, off, 1410 dst_p, dst_q, src, src_cnt, 1411 scf, req_len, flags); 1412 if (!req) 1413 goto fail; 1414 1415 if (first) 1416 sba_chain_request(first, req); 1417 else 1418 first = req; 1419 } 1420 1421 off += req_len; 1422 len -= req_len; 1423 } 1424 1425 return (first) ? &first->tx : NULL; 1426 1427 fail: 1428 if (first) 1429 sba_free_chained_requests(first); 1430 return NULL; 1431 } 1432 1433 /* ====== Mailbox callbacks ===== */ 1434 1435 static void sba_receive_message(struct mbox_client *cl, void *msg) 1436 { 1437 struct brcm_message *m = msg; 1438 struct sba_request *req = m->ctx; 1439 struct sba_device *sba = req->sba; 1440 1441 /* Error count if message has error */ 1442 if (m->error < 0) 1443 dev_err(sba->dev, "%s got message with error %d", 1444 dma_chan_name(&sba->dma_chan), m->error); 1445 1446 /* Process received request */ 1447 sba_process_received_request(sba, req); 1448 } 1449 1450 /* ====== Debugfs callbacks ====== */ 1451 1452 static int sba_debugfs_stats_show(struct seq_file *file, void *offset) 1453 { 1454 struct sba_device *sba = dev_get_drvdata(file->private); 1455 1456 /* Write stats in file */ 1457 sba_write_stats_in_seqfile(sba, file); 1458 1459 return 0; 1460 } 1461 1462 /* ====== Platform driver routines ===== */ 1463 1464 static int sba_prealloc_channel_resources(struct sba_device *sba) 1465 { 1466 int i, j, ret = 0; 1467 struct sba_request *req = NULL; 1468 1469 sba->resp_base = dma_alloc_coherent(sba->mbox_dev, 1470 sba->max_resp_pool_size, 1471 &sba->resp_dma_base, GFP_KERNEL); 1472 if (!sba->resp_base) 1473 return -ENOMEM; 1474 1475 sba->cmds_base = dma_alloc_coherent(sba->mbox_dev, 1476 sba->max_cmds_pool_size, 1477 &sba->cmds_dma_base, GFP_KERNEL); 1478 if (!sba->cmds_base) { 1479 ret = -ENOMEM; 1480 goto fail_free_resp_pool; 1481 } 1482 1483 spin_lock_init(&sba->reqs_lock); 1484 sba->reqs_fence = false; 1485 INIT_LIST_HEAD(&sba->reqs_alloc_list); 1486 INIT_LIST_HEAD(&sba->reqs_pending_list); 1487 INIT_LIST_HEAD(&sba->reqs_active_list); 1488 INIT_LIST_HEAD(&sba->reqs_aborted_list); 1489 INIT_LIST_HEAD(&sba->reqs_free_list); 1490 1491 for (i = 0; i < sba->max_req; i++) { 1492 req = devm_kzalloc(sba->dev, 1493 struct_size(req, cmds, sba->max_cmd_per_req), 1494 GFP_KERNEL); 1495 if (!req) { 1496 ret = -ENOMEM; 1497 goto fail_free_cmds_pool; 1498 } 1499 INIT_LIST_HEAD(&req->node); 1500 req->sba = sba; 1501 req->flags = SBA_REQUEST_STATE_FREE; 1502 INIT_LIST_HEAD(&req->next); 1503 atomic_set(&req->next_pending_count, 0); 1504 for (j = 0; j < sba->max_cmd_per_req; j++) { 1505 req->cmds[j].cmd = 0; 1506 req->cmds[j].cmd_dma = sba->cmds_base + 1507 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1508 req->cmds[j].cmd_dma_addr = sba->cmds_dma_base + 1509 (i * sba->max_cmd_per_req + j) * sizeof(u64); 1510 req->cmds[j].flags = 0; 1511 } 1512 memset(&req->msg, 0, sizeof(req->msg)); 1513 dma_async_tx_descriptor_init(&req->tx, &sba->dma_chan); 1514 async_tx_ack(&req->tx); 1515 req->tx.tx_submit = sba_tx_submit; 1516 req->tx.phys = sba->resp_dma_base + i * sba->hw_resp_size; 1517 list_add_tail(&req->node, &sba->reqs_free_list); 1518 } 1519 1520 return 0; 1521 1522 fail_free_cmds_pool: 1523 dma_free_coherent(sba->mbox_dev, 1524 sba->max_cmds_pool_size, 1525 sba->cmds_base, sba->cmds_dma_base); 1526 fail_free_resp_pool: 1527 dma_free_coherent(sba->mbox_dev, 1528 sba->max_resp_pool_size, 1529 sba->resp_base, sba->resp_dma_base); 1530 return ret; 1531 } 1532 1533 static void sba_freeup_channel_resources(struct sba_device *sba) 1534 { 1535 dmaengine_terminate_all(&sba->dma_chan); 1536 dma_free_coherent(sba->mbox_dev, sba->max_cmds_pool_size, 1537 sba->cmds_base, sba->cmds_dma_base); 1538 dma_free_coherent(sba->mbox_dev, sba->max_resp_pool_size, 1539 sba->resp_base, sba->resp_dma_base); 1540 sba->resp_base = NULL; 1541 sba->resp_dma_base = 0; 1542 } 1543 1544 static int sba_async_register(struct sba_device *sba) 1545 { 1546 int ret; 1547 struct dma_device *dma_dev = &sba->dma_dev; 1548 1549 /* Initialize DMA channel cookie */ 1550 sba->dma_chan.device = dma_dev; 1551 dma_cookie_init(&sba->dma_chan); 1552 1553 /* Initialize DMA device capability mask */ 1554 dma_cap_zero(dma_dev->cap_mask); 1555 dma_cap_set(DMA_INTERRUPT, dma_dev->cap_mask); 1556 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 1557 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 1558 dma_cap_set(DMA_PQ, dma_dev->cap_mask); 1559 1560 /* 1561 * Set mailbox channel device as the base device of 1562 * our dma_device because the actual memory accesses 1563 * will be done by mailbox controller 1564 */ 1565 dma_dev->dev = sba->mbox_dev; 1566 1567 /* Set base prep routines */ 1568 dma_dev->device_free_chan_resources = sba_free_chan_resources; 1569 dma_dev->device_terminate_all = sba_device_terminate_all; 1570 dma_dev->device_issue_pending = sba_issue_pending; 1571 dma_dev->device_tx_status = sba_tx_status; 1572 1573 /* Set interrupt routine */ 1574 if (dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask)) 1575 dma_dev->device_prep_dma_interrupt = sba_prep_dma_interrupt; 1576 1577 /* Set memcpy routine */ 1578 if (dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask)) 1579 dma_dev->device_prep_dma_memcpy = sba_prep_dma_memcpy; 1580 1581 /* Set xor routine and capability */ 1582 if (dma_has_cap(DMA_XOR, dma_dev->cap_mask)) { 1583 dma_dev->device_prep_dma_xor = sba_prep_dma_xor; 1584 dma_dev->max_xor = sba->max_xor_srcs; 1585 } 1586 1587 /* Set pq routine and capability */ 1588 if (dma_has_cap(DMA_PQ, dma_dev->cap_mask)) { 1589 dma_dev->device_prep_dma_pq = sba_prep_dma_pq; 1590 dma_set_maxpq(dma_dev, sba->max_pq_srcs, 0); 1591 } 1592 1593 /* Initialize DMA device channel list */ 1594 INIT_LIST_HEAD(&dma_dev->channels); 1595 list_add_tail(&sba->dma_chan.device_node, &dma_dev->channels); 1596 1597 /* Register with Linux async DMA framework*/ 1598 ret = dma_async_device_register(dma_dev); 1599 if (ret) { 1600 dev_err(sba->dev, "async device register error %d", ret); 1601 return ret; 1602 } 1603 1604 dev_info(sba->dev, "%s capabilities: %s%s%s%s\n", 1605 dma_chan_name(&sba->dma_chan), 1606 dma_has_cap(DMA_INTERRUPT, dma_dev->cap_mask) ? "interrupt " : "", 1607 dma_has_cap(DMA_MEMCPY, dma_dev->cap_mask) ? "memcpy " : "", 1608 dma_has_cap(DMA_XOR, dma_dev->cap_mask) ? "xor " : "", 1609 dma_has_cap(DMA_PQ, dma_dev->cap_mask) ? "pq " : ""); 1610 1611 return 0; 1612 } 1613 1614 static int sba_probe(struct platform_device *pdev) 1615 { 1616 int ret = 0; 1617 struct sba_device *sba; 1618 struct platform_device *mbox_pdev; 1619 struct of_phandle_args args; 1620 1621 /* Allocate main SBA struct */ 1622 sba = devm_kzalloc(&pdev->dev, sizeof(*sba), GFP_KERNEL); 1623 if (!sba) 1624 return -ENOMEM; 1625 1626 sba->dev = &pdev->dev; 1627 platform_set_drvdata(pdev, sba); 1628 1629 /* Number of mailbox channels should be atleast 1 */ 1630 ret = of_count_phandle_with_args(pdev->dev.of_node, 1631 "mboxes", "#mbox-cells"); 1632 if (ret <= 0) 1633 return -ENODEV; 1634 1635 /* Determine SBA version from DT compatible string */ 1636 if (of_device_is_compatible(sba->dev->of_node, "brcm,iproc-sba")) 1637 sba->ver = SBA_VER_1; 1638 else if (of_device_is_compatible(sba->dev->of_node, 1639 "brcm,iproc-sba-v2")) 1640 sba->ver = SBA_VER_2; 1641 else 1642 return -ENODEV; 1643 1644 /* Derived Configuration parameters */ 1645 switch (sba->ver) { 1646 case SBA_VER_1: 1647 sba->hw_buf_size = 4096; 1648 sba->hw_resp_size = 8; 1649 sba->max_pq_coefs = 6; 1650 sba->max_pq_srcs = 6; 1651 break; 1652 case SBA_VER_2: 1653 sba->hw_buf_size = 4096; 1654 sba->hw_resp_size = 8; 1655 sba->max_pq_coefs = 30; 1656 /* 1657 * We can support max_pq_srcs == max_pq_coefs because 1658 * we are limited by number of SBA commands that we can 1659 * fit in one message for underlying ring manager HW. 1660 */ 1661 sba->max_pq_srcs = 12; 1662 break; 1663 default: 1664 return -EINVAL; 1665 } 1666 sba->max_req = SBA_MAX_REQ_PER_MBOX_CHANNEL; 1667 sba->max_cmd_per_req = sba->max_pq_srcs + 3; 1668 sba->max_xor_srcs = sba->max_cmd_per_req - 1; 1669 sba->max_resp_pool_size = sba->max_req * sba->hw_resp_size; 1670 sba->max_cmds_pool_size = sba->max_req * 1671 sba->max_cmd_per_req * sizeof(u64); 1672 1673 /* Setup mailbox client */ 1674 sba->client.dev = &pdev->dev; 1675 sba->client.rx_callback = sba_receive_message; 1676 sba->client.tx_block = false; 1677 sba->client.knows_txdone = true; 1678 sba->client.tx_tout = 0; 1679 1680 /* Request mailbox channel */ 1681 sba->mchan = mbox_request_channel(&sba->client, 0); 1682 if (IS_ERR(sba->mchan)) { 1683 ret = PTR_ERR(sba->mchan); 1684 goto fail_free_mchan; 1685 } 1686 1687 /* Find-out underlying mailbox device */ 1688 ret = of_parse_phandle_with_args(pdev->dev.of_node, 1689 "mboxes", "#mbox-cells", 0, &args); 1690 if (ret) 1691 goto fail_free_mchan; 1692 mbox_pdev = of_find_device_by_node(args.np); 1693 of_node_put(args.np); 1694 if (!mbox_pdev) { 1695 ret = -ENODEV; 1696 goto fail_free_mchan; 1697 } 1698 sba->mbox_dev = &mbox_pdev->dev; 1699 1700 /* Prealloc channel resource */ 1701 ret = sba_prealloc_channel_resources(sba); 1702 if (ret) 1703 goto fail_put_mbox; 1704 1705 /* Check availability of debugfs */ 1706 if (!debugfs_initialized()) 1707 goto skip_debugfs; 1708 1709 /* Create debugfs root entry */ 1710 sba->root = debugfs_create_dir(dev_name(sba->dev), NULL); 1711 1712 /* Create debugfs stats entry */ 1713 debugfs_create_devm_seqfile(sba->dev, "stats", sba->root, 1714 sba_debugfs_stats_show); 1715 1716 skip_debugfs: 1717 1718 /* Register DMA device with Linux async framework */ 1719 ret = sba_async_register(sba); 1720 if (ret) 1721 goto fail_free_resources; 1722 1723 /* Print device info */ 1724 dev_info(sba->dev, "%s using SBAv%d mailbox channel from %s", 1725 dma_chan_name(&sba->dma_chan), sba->ver+1, 1726 dev_name(sba->mbox_dev)); 1727 1728 return 0; 1729 1730 fail_free_resources: 1731 debugfs_remove_recursive(sba->root); 1732 sba_freeup_channel_resources(sba); 1733 fail_put_mbox: 1734 put_device(sba->mbox_dev); 1735 fail_free_mchan: 1736 mbox_free_channel(sba->mchan); 1737 return ret; 1738 } 1739 1740 static void sba_remove(struct platform_device *pdev) 1741 { 1742 struct sba_device *sba = platform_get_drvdata(pdev); 1743 1744 dma_async_device_unregister(&sba->dma_dev); 1745 1746 debugfs_remove_recursive(sba->root); 1747 1748 sba_freeup_channel_resources(sba); 1749 1750 put_device(sba->mbox_dev); 1751 1752 mbox_free_channel(sba->mchan); 1753 } 1754 1755 static const struct of_device_id sba_of_match[] = { 1756 { .compatible = "brcm,iproc-sba", }, 1757 { .compatible = "brcm,iproc-sba-v2", }, 1758 {}, 1759 }; 1760 MODULE_DEVICE_TABLE(of, sba_of_match); 1761 1762 static struct platform_driver sba_driver = { 1763 .probe = sba_probe, 1764 .remove = sba_remove, 1765 .driver = { 1766 .name = "bcm-sba-raid", 1767 .of_match_table = sba_of_match, 1768 }, 1769 }; 1770 module_platform_driver(sba_driver); 1771 1772 MODULE_DESCRIPTION("Broadcom SBA RAID driver"); 1773 MODULE_AUTHOR("Anup Patel <anup.patel@broadcom.com>"); 1774 MODULE_LICENSE("GPL v2"); 1775