1 /* 2 * drivers/dma/fsl_raid.c 3 * 4 * Freescale RAID Engine device driver 5 * 6 * Author: 7 * Harninder Rai <harninder.rai@freescale.com> 8 * Naveen Burmi <naveenburmi@freescale.com> 9 * 10 * Rewrite: 11 * Xuelin Shi <xuelin.shi@freescale.com> 12 * 13 * Copyright (c) 2010-2014 Freescale Semiconductor, Inc. 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions are met: 17 * * Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * * Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * * Neither the name of Freescale Semiconductor nor the 23 * names of its contributors may be used to endorse or promote products 24 * derived from this software without specific prior written permission. 25 * 26 * ALTERNATIVELY, this software may be distributed under the terms of the 27 * GNU General Public License ("GPL") as published by the Free Software 28 * Foundation, either version 2 of that License or (at your option) any 29 * later version. 30 * 31 * THIS SOFTWARE IS PROVIDED BY Freescale Semiconductor ``AS IS'' AND ANY 32 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 33 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 34 * DISCLAIMED. IN NO EVENT SHALL Freescale Semiconductor BE LIABLE FOR ANY 35 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 36 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 37 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 38 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 39 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 40 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 41 * 42 * Theory of operation: 43 * 44 * General capabilities: 45 * RAID Engine (RE) block is capable of offloading XOR, memcpy and P/Q 46 * calculations required in RAID5 and RAID6 operations. RE driver 47 * registers with Linux's ASYNC layer as dma driver. RE hardware 48 * maintains strict ordering of the requests through chained 49 * command queueing. 50 * 51 * Data flow: 52 * Software RAID layer of Linux (MD layer) maintains RAID partitions, 53 * strips, stripes etc. It sends requests to the underlying ASYNC layer 54 * which further passes it to RE driver. ASYNC layer decides which request 55 * goes to which job ring of RE hardware. For every request processed by 56 * RAID Engine, driver gets an interrupt unless coalescing is set. The 57 * per job ring interrupt handler checks the status register for errors, 58 * clears the interrupt and leave the post interrupt processing to the irq 59 * thread. 60 */ 61 #include <linux/interrupt.h> 62 #include <linux/module.h> 63 #include <linux/of.h> 64 #include <linux/of_irq.h> 65 #include <linux/of_platform.h> 66 #include <linux/platform_device.h> 67 #include <linux/dma-mapping.h> 68 #include <linux/dmapool.h> 69 #include <linux/dmaengine.h> 70 #include <linux/io.h> 71 #include <linux/spinlock.h> 72 #include <linux/slab.h> 73 74 #include "dmaengine.h" 75 #include "fsl_raid.h" 76 77 #define FSL_RE_MAX_XOR_SRCS 16 78 #define FSL_RE_MAX_PQ_SRCS 16 79 #define FSL_RE_MIN_DESCS 256 80 #define FSL_RE_MAX_DESCS (4 * FSL_RE_MIN_DESCS) 81 #define FSL_RE_FRAME_FORMAT 0x1 82 #define FSL_RE_MAX_DATA_LEN (1024*1024) 83 84 #define to_fsl_re_dma_desc(tx) container_of(tx, struct fsl_re_desc, async_tx) 85 86 /* Add descriptors into per chan software queue - submit_q */ 87 static dma_cookie_t fsl_re_tx_submit(struct dma_async_tx_descriptor *tx) 88 { 89 struct fsl_re_desc *desc; 90 struct fsl_re_chan *re_chan; 91 dma_cookie_t cookie; 92 unsigned long flags; 93 94 desc = to_fsl_re_dma_desc(tx); 95 re_chan = container_of(tx->chan, struct fsl_re_chan, chan); 96 97 spin_lock_irqsave(&re_chan->desc_lock, flags); 98 cookie = dma_cookie_assign(tx); 99 list_add_tail(&desc->node, &re_chan->submit_q); 100 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 101 102 return cookie; 103 } 104 105 /* Copy descriptor from per chan software queue into hardware job ring */ 106 static void fsl_re_issue_pending(struct dma_chan *chan) 107 { 108 struct fsl_re_chan *re_chan; 109 int avail; 110 struct fsl_re_desc *desc, *_desc; 111 unsigned long flags; 112 113 re_chan = container_of(chan, struct fsl_re_chan, chan); 114 115 spin_lock_irqsave(&re_chan->desc_lock, flags); 116 avail = FSL_RE_SLOT_AVAIL( 117 in_be32(&re_chan->jrregs->inbring_slot_avail)); 118 119 list_for_each_entry_safe(desc, _desc, &re_chan->submit_q, node) { 120 if (!avail) 121 break; 122 123 list_move_tail(&desc->node, &re_chan->active_q); 124 125 memcpy(&re_chan->inb_ring_virt_addr[re_chan->inb_count], 126 &desc->hwdesc, sizeof(struct fsl_re_hw_desc)); 127 128 re_chan->inb_count = (re_chan->inb_count + 1) & 129 FSL_RE_RING_SIZE_MASK; 130 out_be32(&re_chan->jrregs->inbring_add_job, FSL_RE_ADD_JOB(1)); 131 avail--; 132 } 133 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 134 } 135 136 static void fsl_re_desc_done(struct fsl_re_desc *desc) 137 { 138 dma_cookie_complete(&desc->async_tx); 139 dma_descriptor_unmap(&desc->async_tx); 140 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 141 } 142 143 static void fsl_re_cleanup_descs(struct fsl_re_chan *re_chan) 144 { 145 struct fsl_re_desc *desc, *_desc; 146 unsigned long flags; 147 148 spin_lock_irqsave(&re_chan->desc_lock, flags); 149 list_for_each_entry_safe(desc, _desc, &re_chan->ack_q, node) { 150 if (async_tx_test_ack(&desc->async_tx)) 151 list_move_tail(&desc->node, &re_chan->free_q); 152 } 153 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 154 155 fsl_re_issue_pending(&re_chan->chan); 156 } 157 158 static void fsl_re_dequeue(struct tasklet_struct *t) 159 { 160 struct fsl_re_chan *re_chan = from_tasklet(re_chan, t, irqtask); 161 struct fsl_re_desc *desc, *_desc; 162 struct fsl_re_hw_desc *hwdesc; 163 unsigned long flags; 164 unsigned int count, oub_count; 165 LIST_HEAD(completed); 166 int found; 167 168 fsl_re_cleanup_descs(re_chan); 169 170 spin_lock_irqsave(&re_chan->desc_lock, flags); 171 count = FSL_RE_SLOT_FULL(in_be32(&re_chan->jrregs->oubring_slot_full)); 172 while (count--) { 173 found = 0; 174 hwdesc = &re_chan->oub_ring_virt_addr[re_chan->oub_count]; 175 list_for_each_entry_safe(desc, _desc, &re_chan->active_q, 176 node) { 177 /* compare the hw dma addr to find the completed */ 178 if (desc->hwdesc.lbea32 == hwdesc->lbea32 && 179 desc->hwdesc.addr_low == hwdesc->addr_low) { 180 found = 1; 181 break; 182 } 183 } 184 185 if (found) { 186 list_move_tail(&desc->node, &completed); 187 } else { 188 dev_err(re_chan->dev, 189 "found hwdesc not in sw queue, discard it\n"); 190 } 191 192 oub_count = (re_chan->oub_count + 1) & FSL_RE_RING_SIZE_MASK; 193 re_chan->oub_count = oub_count; 194 195 out_be32(&re_chan->jrregs->oubring_job_rmvd, 196 FSL_RE_RMVD_JOB(1)); 197 } 198 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 199 200 /* Invoke the client callbacks outside the channel lock. The callback 201 * may submit new work which re-acquires desc_lock, so holding it here 202 * would deadlock. 203 */ 204 list_for_each_entry_safe(desc, _desc, &completed, node) { 205 fsl_re_desc_done(desc); 206 spin_lock_irqsave(&re_chan->desc_lock, flags); 207 list_move_tail(&desc->node, &re_chan->ack_q); 208 spin_unlock_irqrestore(&re_chan->desc_lock, flags); 209 } 210 } 211 212 /* Per Job Ring interrupt handler */ 213 static irqreturn_t fsl_re_isr(int irq, void *data) 214 { 215 struct fsl_re_chan *re_chan; 216 u32 irqstate, status; 217 218 re_chan = dev_get_drvdata((struct device *)data); 219 220 irqstate = in_be32(&re_chan->jrregs->jr_interrupt_status); 221 if (!irqstate) 222 return IRQ_NONE; 223 224 /* 225 * There's no way in upper layer (read MD layer) to recover from 226 * error conditions except restart everything. In long term we 227 * need to do something more than just crashing 228 */ 229 if (irqstate & FSL_RE_ERROR) { 230 status = in_be32(&re_chan->jrregs->jr_status); 231 dev_err(re_chan->dev, "chan error irqstate: %x, status: %x\n", 232 irqstate, status); 233 } 234 235 /* Clear interrupt */ 236 out_be32(&re_chan->jrregs->jr_interrupt_status, FSL_RE_CLR_INTR); 237 238 tasklet_schedule(&re_chan->irqtask); 239 240 return IRQ_HANDLED; 241 } 242 243 static enum dma_status fsl_re_tx_status(struct dma_chan *chan, 244 dma_cookie_t cookie, 245 struct dma_tx_state *txstate) 246 { 247 return dma_cookie_status(chan, cookie, txstate); 248 } 249 250 static void fill_cfd_frame(struct fsl_re_cmpnd_frame *cf, u8 index, 251 size_t length, dma_addr_t addr, bool final) 252 { 253 u32 efrl = length & FSL_RE_CF_LENGTH_MASK; 254 255 efrl |= final << FSL_RE_CF_FINAL_SHIFT; 256 cf[index].efrl32 = cpu_to_be32(efrl); 257 cf[index].addr_high = cpu_to_be32(upper_32_bits(addr)); 258 cf[index].addr_low = cpu_to_be32(lower_32_bits(addr)); 259 } 260 261 static struct fsl_re_desc *fsl_re_init_desc(struct fsl_re_chan *re_chan, 262 struct fsl_re_desc *desc, 263 void *cf, dma_addr_t paddr) 264 { 265 desc->re_chan = re_chan; 266 desc->async_tx.tx_submit = fsl_re_tx_submit; 267 dma_async_tx_descriptor_init(&desc->async_tx, &re_chan->chan); 268 INIT_LIST_HEAD(&desc->node); 269 270 desc->hwdesc.fmt32 = cpu_to_be32(FSL_RE_FRAME_FORMAT << 271 FSL_RE_HWDESC_FMT_SHIFT); 272 desc->hwdesc.lbea32 = cpu_to_be32(upper_32_bits(paddr)); 273 desc->hwdesc.addr_low = cpu_to_be32(lower_32_bits(paddr)); 274 desc->cf_addr = cf; 275 desc->cf_paddr = paddr; 276 277 desc->cdb_addr = (void *)(cf + FSL_RE_CF_DESC_SIZE); 278 desc->cdb_paddr = paddr + FSL_RE_CF_DESC_SIZE; 279 280 return desc; 281 } 282 283 static struct fsl_re_desc *fsl_re_chan_alloc_desc(struct fsl_re_chan *re_chan, 284 unsigned long flags) 285 { 286 struct fsl_re_desc *desc = NULL; 287 void *cf; 288 dma_addr_t paddr; 289 unsigned long lock_flag; 290 291 fsl_re_cleanup_descs(re_chan); 292 293 spin_lock_irqsave(&re_chan->desc_lock, lock_flag); 294 if (!list_empty(&re_chan->free_q)) { 295 /* take one desc from free_q */ 296 desc = list_first_entry(&re_chan->free_q, 297 struct fsl_re_desc, node); 298 list_del(&desc->node); 299 300 desc->async_tx.flags = flags; 301 } 302 spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); 303 304 if (!desc) { 305 desc = kzalloc_obj(*desc, GFP_NOWAIT); 306 if (!desc) 307 return NULL; 308 309 cf = dma_pool_zalloc(re_chan->re_dev->cf_desc_pool, GFP_NOWAIT, 310 &paddr); 311 if (!cf) { 312 kfree(desc); 313 return NULL; 314 } 315 316 desc = fsl_re_init_desc(re_chan, desc, cf, paddr); 317 desc->async_tx.flags = flags; 318 319 spin_lock_irqsave(&re_chan->desc_lock, lock_flag); 320 re_chan->alloc_count++; 321 spin_unlock_irqrestore(&re_chan->desc_lock, lock_flag); 322 } 323 324 return desc; 325 } 326 327 static struct dma_async_tx_descriptor *fsl_re_prep_dma_genq( 328 struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 329 unsigned int src_cnt, const unsigned char *scf, size_t len, 330 unsigned long flags) 331 { 332 struct fsl_re_chan *re_chan; 333 struct fsl_re_desc *desc; 334 struct fsl_re_xor_cdb *xor; 335 struct fsl_re_cmpnd_frame *cf; 336 u32 cdb; 337 unsigned int i, j; 338 unsigned int save_src_cnt = src_cnt; 339 int cont_q = 0; 340 341 re_chan = container_of(chan, struct fsl_re_chan, chan); 342 if (len > FSL_RE_MAX_DATA_LEN) { 343 dev_err(re_chan->dev, "genq tx length %zu, max length %d\n", 344 len, FSL_RE_MAX_DATA_LEN); 345 return NULL; 346 } 347 348 desc = fsl_re_chan_alloc_desc(re_chan, flags); 349 if (desc <= 0) 350 return NULL; 351 352 if (scf && (flags & DMA_PREP_CONTINUE)) { 353 cont_q = 1; 354 src_cnt += 1; 355 } 356 357 /* Filling xor CDB */ 358 cdb = FSL_RE_XOR_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; 359 cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; 360 cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; 361 cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; 362 cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 363 xor = desc->cdb_addr; 364 xor->cdb32 = cpu_to_be32(cdb); 365 366 if (scf) { 367 /* compute q = src0*coef0^src1*coef1^..., * is GF(8) mult */ 368 for (i = 0; i < save_src_cnt; i++) 369 xor->gfm[i] = scf[i]; 370 if (cont_q) 371 xor->gfm[i++] = 1; 372 } else { 373 /* compute P, that is XOR all srcs */ 374 for (i = 0; i < src_cnt; i++) 375 xor->gfm[i] = 1; 376 } 377 378 /* Filling frame 0 of compound frame descriptor with CDB */ 379 cf = desc->cf_addr; 380 fill_cfd_frame(cf, 0, sizeof(*xor), desc->cdb_paddr, 0); 381 382 /* Fill CFD's 1st frame with dest buffer */ 383 fill_cfd_frame(cf, 1, len, dest, 0); 384 385 /* Fill CFD's rest of the frames with source buffers */ 386 for (i = 2, j = 0; j < save_src_cnt; i++, j++) 387 fill_cfd_frame(cf, i, len, src[j], 0); 388 389 /* Fill the last frame and mark it final */ 390 if (cont_q) 391 fill_cfd_frame(cf, i, len, dest, 1); 392 else 393 fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); 394 395 return &desc->async_tx; 396 } 397 398 /* 399 * Prep function for P parity calculation.In RAID Engine terminology, 400 * XOR calculation is called GenQ calculation done through GenQ command 401 */ 402 static struct dma_async_tx_descriptor *fsl_re_prep_dma_xor( 403 struct dma_chan *chan, dma_addr_t dest, dma_addr_t *src, 404 unsigned int src_cnt, size_t len, unsigned long flags) 405 { 406 /* NULL let genq take all coef as 1 */ 407 return fsl_re_prep_dma_genq(chan, dest, src, src_cnt, NULL, len, flags); 408 } 409 410 /* 411 * Prep function for P/Q parity calculation.In RAID Engine terminology, 412 * P/Q calculation is called GenQQ done through GenQQ command 413 */ 414 static struct dma_async_tx_descriptor *fsl_re_prep_dma_pq( 415 struct dma_chan *chan, dma_addr_t *dest, dma_addr_t *src, 416 unsigned int src_cnt, const unsigned char *scf, size_t len, 417 unsigned long flags) 418 { 419 struct fsl_re_chan *re_chan; 420 struct fsl_re_desc *desc; 421 struct fsl_re_pq_cdb *pq; 422 struct fsl_re_cmpnd_frame *cf; 423 u32 cdb; 424 u8 *p; 425 int gfmq_len, i, j; 426 unsigned int save_src_cnt = src_cnt; 427 428 re_chan = container_of(chan, struct fsl_re_chan, chan); 429 if (len > FSL_RE_MAX_DATA_LEN) { 430 dev_err(re_chan->dev, "pq tx length is %zu, max length is %d\n", 431 len, FSL_RE_MAX_DATA_LEN); 432 return NULL; 433 } 434 435 /* 436 * RE requires at least 2 sources, if given only one source, we pass the 437 * second source same as the first one. 438 * With only one source, generating P is meaningless, only generate Q. 439 */ 440 if (src_cnt == 1) { 441 struct dma_async_tx_descriptor *tx; 442 dma_addr_t dma_src[2]; 443 unsigned char coef[2]; 444 445 dma_src[0] = *src; 446 coef[0] = *scf; 447 dma_src[1] = *src; 448 coef[1] = 0; 449 tx = fsl_re_prep_dma_genq(chan, dest[1], dma_src, 2, coef, len, 450 flags); 451 if (tx) 452 desc = to_fsl_re_dma_desc(tx); 453 454 return tx; 455 } 456 457 /* 458 * During RAID6 array creation, Linux's MD layer gets P and Q 459 * calculated separately in two steps. But our RAID Engine has 460 * the capability to calculate both P and Q with a single command 461 * Hence to merge well with MD layer, we need to provide a hook 462 * here and call re_jq_prep_dma_genq() function 463 */ 464 465 if (flags & DMA_PREP_PQ_DISABLE_P) 466 return fsl_re_prep_dma_genq(chan, dest[1], src, src_cnt, 467 scf, len, flags); 468 469 if (flags & DMA_PREP_CONTINUE) 470 src_cnt += 3; 471 472 desc = fsl_re_chan_alloc_desc(re_chan, flags); 473 if (desc <= 0) 474 return NULL; 475 476 /* Filling GenQQ CDB */ 477 cdb = FSL_RE_PQ_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; 478 cdb |= (src_cnt - 1) << FSL_RE_CDB_NRCS_SHIFT; 479 cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; 480 cdb |= FSL_RE_BUFFER_OUTPUT << FSL_RE_CDB_BUFFER_SHIFT; 481 cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 482 483 pq = desc->cdb_addr; 484 pq->cdb32 = cpu_to_be32(cdb); 485 486 p = pq->gfm_q1; 487 /* Init gfm_q1[] */ 488 for (i = 0; i < src_cnt; i++) 489 p[i] = 1; 490 491 /* Align gfm[] to 32bit */ 492 gfmq_len = ALIGN(src_cnt, 4); 493 494 /* Init gfm_q2[] */ 495 p += gfmq_len; 496 for (i = 0; i < src_cnt; i++) 497 p[i] = scf[i]; 498 499 /* Filling frame 0 of compound frame descriptor with CDB */ 500 cf = desc->cf_addr; 501 fill_cfd_frame(cf, 0, sizeof(struct fsl_re_pq_cdb), desc->cdb_paddr, 0); 502 503 /* Fill CFD's 1st & 2nd frame with dest buffers */ 504 for (i = 1, j = 0; i < 3; i++, j++) 505 fill_cfd_frame(cf, i, len, dest[j], 0); 506 507 /* Fill CFD's rest of the frames with source buffers */ 508 for (i = 3, j = 0; j < save_src_cnt; i++, j++) 509 fill_cfd_frame(cf, i, len, src[j], 0); 510 511 /* PQ computation continuation */ 512 if (flags & DMA_PREP_CONTINUE) { 513 if (src_cnt - save_src_cnt == 3) { 514 p[save_src_cnt] = 0; 515 p[save_src_cnt + 1] = 0; 516 p[save_src_cnt + 2] = 1; 517 fill_cfd_frame(cf, i++, len, dest[0], 0); 518 fill_cfd_frame(cf, i++, len, dest[1], 0); 519 fill_cfd_frame(cf, i++, len, dest[1], 1); 520 } else { 521 dev_err(re_chan->dev, "PQ tx continuation error!\n"); 522 return NULL; 523 } 524 } else { 525 /* Mark the last source buffer frame final */ 526 fill_cfd_frame(cf, i - 1, len, src[j - 1], 1); 527 } 528 529 return &desc->async_tx; 530 } 531 532 /* 533 * Prep function for memcpy. In RAID Engine, memcpy is done through MOVE 534 * command. Logic of this function will need to be modified once multipage 535 * support is added in Linux's MD/ASYNC Layer 536 */ 537 static struct dma_async_tx_descriptor *fsl_re_prep_dma_memcpy( 538 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src, 539 size_t len, unsigned long flags) 540 { 541 struct fsl_re_chan *re_chan; 542 struct fsl_re_desc *desc; 543 size_t length; 544 struct fsl_re_cmpnd_frame *cf; 545 struct fsl_re_move_cdb *move; 546 u32 cdb; 547 548 re_chan = container_of(chan, struct fsl_re_chan, chan); 549 550 if (len > FSL_RE_MAX_DATA_LEN) { 551 dev_err(re_chan->dev, "cp tx length is %zu, max length is %d\n", 552 len, FSL_RE_MAX_DATA_LEN); 553 return NULL; 554 } 555 556 desc = fsl_re_chan_alloc_desc(re_chan, flags); 557 if (desc <= 0) 558 return NULL; 559 560 /* Filling move CDB */ 561 cdb = FSL_RE_MOVE_OPCODE << FSL_RE_CDB_OPCODE_SHIFT; 562 cdb |= FSL_RE_BLOCK_SIZE << FSL_RE_CDB_BLKSIZE_SHIFT; 563 cdb |= FSL_RE_INTR_ON_ERROR << FSL_RE_CDB_ERROR_SHIFT; 564 cdb |= FSL_RE_DATA_DEP << FSL_RE_CDB_DEPEND_SHIFT; 565 566 move = desc->cdb_addr; 567 move->cdb32 = cpu_to_be32(cdb); 568 569 /* Filling frame 0 of CFD with move CDB */ 570 cf = desc->cf_addr; 571 fill_cfd_frame(cf, 0, sizeof(*move), desc->cdb_paddr, 0); 572 573 length = min_t(size_t, len, FSL_RE_MAX_DATA_LEN); 574 575 /* Fill CFD's 1st frame with dest buffer */ 576 fill_cfd_frame(cf, 1, length, dest, 0); 577 578 /* Fill CFD's 2nd frame with src buffer */ 579 fill_cfd_frame(cf, 2, length, src, 1); 580 581 return &desc->async_tx; 582 } 583 584 static int fsl_re_alloc_chan_resources(struct dma_chan *chan) 585 { 586 struct fsl_re_chan *re_chan; 587 struct fsl_re_desc *desc; 588 void *cf; 589 dma_addr_t paddr; 590 int i; 591 592 re_chan = container_of(chan, struct fsl_re_chan, chan); 593 for (i = 0; i < FSL_RE_MIN_DESCS; i++) { 594 desc = kzalloc_obj(*desc); 595 if (!desc) 596 break; 597 598 cf = dma_pool_zalloc(re_chan->re_dev->cf_desc_pool, GFP_KERNEL, 599 &paddr); 600 if (!cf) { 601 kfree(desc); 602 break; 603 } 604 605 INIT_LIST_HEAD(&desc->node); 606 fsl_re_init_desc(re_chan, desc, cf, paddr); 607 608 list_add_tail(&desc->node, &re_chan->free_q); 609 re_chan->alloc_count++; 610 } 611 return re_chan->alloc_count; 612 } 613 614 static void fsl_re_free_chan_resources(struct dma_chan *chan) 615 { 616 struct fsl_re_chan *re_chan; 617 struct fsl_re_desc *desc, *_desc; 618 619 re_chan = container_of(chan, struct fsl_re_chan, chan); 620 list_for_each_entry_safe(desc, _desc, &re_chan->free_q, node) { 621 list_del(&desc->node); 622 dma_pool_free(re_chan->re_dev->cf_desc_pool, desc->cf_addr, 623 desc->cf_paddr); 624 kfree(desc); 625 re_chan->alloc_count--; 626 } 627 628 if (!list_empty(&re_chan->free_q)) 629 dev_err(re_chan->dev, "chan resource cannot be cleaned!\n"); 630 } 631 632 static int fsl_re_chan_probe(struct platform_device *ofdev, 633 struct device_node *np, u8 q, u32 off) 634 { 635 struct device *dev, *chandev; 636 struct fsl_re_drv_private *re_priv; 637 struct fsl_re_chan *chan; 638 struct dma_device *dma_dev; 639 u32 ptr; 640 u32 status; 641 int ret = 0, rc; 642 struct platform_device *chan_ofdev; 643 644 dev = &ofdev->dev; 645 re_priv = dev_get_drvdata(dev); 646 dma_dev = &re_priv->dma_dev; 647 648 chan = devm_kzalloc(dev, sizeof(*chan), GFP_KERNEL); 649 if (!chan) 650 return -ENOMEM; 651 652 /* create platform device for chan node */ 653 chan_ofdev = of_platform_device_create(np, NULL, dev); 654 if (!chan_ofdev) { 655 dev_err(dev, "Not able to create ofdev for jr %d\n", q); 656 ret = -EINVAL; 657 goto err_free; 658 } 659 660 /* read reg property from dts */ 661 rc = of_property_read_u32(np, "reg", &ptr); 662 if (rc) { 663 dev_err(dev, "Reg property not found in jr %d\n", q); 664 ret = -ENODEV; 665 goto err_free; 666 } 667 668 chan->jrregs = re_priv->base + off + ptr; 669 670 /* read irq property from dts */ 671 chan->irq = irq_of_parse_and_map(np, 0); 672 if (!chan->irq) { 673 dev_err(dev, "No IRQ defined for JR %d\n", q); 674 ret = -ENODEV; 675 goto err_free; 676 } 677 678 snprintf(chan->name, sizeof(chan->name), "re_jr%02d", q); 679 680 chandev = &chan_ofdev->dev; 681 tasklet_setup(&chan->irqtask, fsl_re_dequeue); 682 683 ret = request_irq(chan->irq, fsl_re_isr, 0, chan->name, chandev); 684 if (ret) { 685 dev_err(dev, "Unable to register interrupt for JR %d\n", q); 686 ret = -EINVAL; 687 goto err_free; 688 } 689 690 re_priv->re_jrs[q] = chan; 691 chan->chan.device = dma_dev; 692 chan->chan.private = chan; 693 chan->dev = chandev; 694 chan->re_dev = re_priv; 695 696 spin_lock_init(&chan->desc_lock); 697 INIT_LIST_HEAD(&chan->ack_q); 698 INIT_LIST_HEAD(&chan->active_q); 699 INIT_LIST_HEAD(&chan->submit_q); 700 INIT_LIST_HEAD(&chan->free_q); 701 702 chan->inb_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool, 703 GFP_KERNEL, &chan->inb_phys_addr); 704 if (!chan->inb_ring_virt_addr) { 705 dev_err(dev, "No dma memory for inb_ring_virt_addr\n"); 706 ret = -ENOMEM; 707 goto err_free; 708 } 709 710 chan->oub_ring_virt_addr = dma_pool_alloc(chan->re_dev->hw_desc_pool, 711 GFP_KERNEL, &chan->oub_phys_addr); 712 if (!chan->oub_ring_virt_addr) { 713 dev_err(dev, "No dma memory for oub_ring_virt_addr\n"); 714 ret = -ENOMEM; 715 goto err_free_1; 716 } 717 718 /* Program the Inbound/Outbound ring base addresses and size */ 719 out_be32(&chan->jrregs->inbring_base_h, 720 chan->inb_phys_addr & FSL_RE_ADDR_BIT_MASK); 721 out_be32(&chan->jrregs->oubring_base_h, 722 chan->oub_phys_addr & FSL_RE_ADDR_BIT_MASK); 723 out_be32(&chan->jrregs->inbring_base_l, 724 chan->inb_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); 725 out_be32(&chan->jrregs->oubring_base_l, 726 chan->oub_phys_addr >> FSL_RE_ADDR_BIT_SHIFT); 727 out_be32(&chan->jrregs->inbring_size, 728 FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); 729 out_be32(&chan->jrregs->oubring_size, 730 FSL_RE_RING_SIZE << FSL_RE_RING_SIZE_SHIFT); 731 732 /* Read LIODN value from u-boot */ 733 status = in_be32(&chan->jrregs->jr_config_1) & FSL_RE_REG_LIODN_MASK; 734 735 /* Program the CFG reg */ 736 out_be32(&chan->jrregs->jr_config_1, 737 FSL_RE_CFG1_CBSI | FSL_RE_CFG1_CBS0 | status); 738 739 dev_set_drvdata(chandev, chan); 740 741 /* Enable RE/CHAN */ 742 out_be32(&chan->jrregs->jr_command, FSL_RE_ENABLE); 743 744 return 0; 745 746 err_free_1: 747 dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, 748 chan->inb_phys_addr); 749 err_free: 750 return ret; 751 } 752 753 /* Probe function for RAID Engine */ 754 static int fsl_re_probe(struct platform_device *ofdev) 755 { 756 struct fsl_re_ctrl __iomem *re_regs; 757 struct fsl_re_drv_private *re_priv; 758 struct device_node *child; 759 u32 off; 760 u8 ridx = 0; 761 struct dma_device *dma_dev; 762 int rc; 763 struct device *dev = &ofdev->dev; 764 765 /* IOMAP the entire RAID Engine region */ 766 re_regs = devm_platform_ioremap_resource(ofdev, 0); 767 if (IS_ERR(re_regs)) 768 return PTR_ERR(re_regs); 769 770 re_priv = devm_kzalloc(dev, sizeof(*re_priv), GFP_KERNEL); 771 if (!re_priv) 772 return -ENOMEM; 773 774 re_priv->base = re_regs; 775 776 /* Program the RE mode */ 777 out_be32(&re_regs->global_config, FSL_RE_NON_DPAA_MODE); 778 779 /* Program Galois Field polynomial */ 780 out_be32(&re_regs->galois_field_config, FSL_RE_GFM_POLY); 781 782 dev_info(dev, "version %x, mode %x, gfp %x\n", 783 in_be32(&re_regs->re_version_id), 784 in_be32(&re_regs->global_config), 785 in_be32(&re_regs->galois_field_config)); 786 787 dma_dev = &re_priv->dma_dev; 788 dma_dev->dev = dev; 789 INIT_LIST_HEAD(&dma_dev->channels); 790 dma_set_mask(dev, DMA_BIT_MASK(40)); 791 792 dma_dev->device_alloc_chan_resources = fsl_re_alloc_chan_resources; 793 dma_dev->device_tx_status = fsl_re_tx_status; 794 dma_dev->device_issue_pending = fsl_re_issue_pending; 795 796 dma_dev->max_xor = FSL_RE_MAX_XOR_SRCS; 797 dma_dev->device_prep_dma_xor = fsl_re_prep_dma_xor; 798 dma_cap_set(DMA_XOR, dma_dev->cap_mask); 799 800 dma_dev->max_pq = FSL_RE_MAX_PQ_SRCS; 801 dma_dev->device_prep_dma_pq = fsl_re_prep_dma_pq; 802 dma_cap_set(DMA_PQ, dma_dev->cap_mask); 803 804 dma_dev->device_prep_dma_memcpy = fsl_re_prep_dma_memcpy; 805 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask); 806 807 dma_dev->device_free_chan_resources = fsl_re_free_chan_resources; 808 809 re_priv->total_chans = 0; 810 811 re_priv->cf_desc_pool = dmam_pool_create("fsl_re_cf_desc_pool", dev, 812 FSL_RE_CF_CDB_SIZE, 813 FSL_RE_CF_CDB_ALIGN, 0); 814 815 if (!re_priv->cf_desc_pool) { 816 dev_err(dev, "No memory for fsl re_cf desc pool\n"); 817 return -ENOMEM; 818 } 819 820 re_priv->hw_desc_pool = dmam_pool_create("fsl_re_hw_desc_pool", dev, 821 sizeof(struct fsl_re_hw_desc) * FSL_RE_RING_SIZE, 822 FSL_RE_FRAME_ALIGN, 0); 823 if (!re_priv->hw_desc_pool) { 824 dev_err(dev, "No memory for fsl re_hw desc pool\n"); 825 return -ENOMEM; 826 } 827 828 dev_set_drvdata(dev, re_priv); 829 830 /* Parse Device tree to find out the total number of JQs present */ 831 for_each_compatible_node_scoped(np, NULL, "fsl,raideng-v1.0-job-queue") { 832 rc = of_property_read_u32(np, "reg", &off); 833 if (rc) { 834 dev_err(dev, "Reg property not found in JQ node\n"); 835 return -ENODEV; 836 } 837 /* Find out the Job Rings present under each JQ */ 838 for_each_child_of_node(np, child) { 839 rc = of_device_is_compatible(child, 840 "fsl,raideng-v1.0-job-ring"); 841 if (rc) { 842 fsl_re_chan_probe(ofdev, child, ridx++, off); 843 re_priv->total_chans++; 844 } 845 } 846 } 847 848 dma_async_device_register(dma_dev); 849 850 return 0; 851 } 852 853 static void fsl_re_remove_chan(struct fsl_re_chan *chan) 854 { 855 tasklet_kill(&chan->irqtask); 856 857 dma_pool_free(chan->re_dev->hw_desc_pool, chan->inb_ring_virt_addr, 858 chan->inb_phys_addr); 859 860 dma_pool_free(chan->re_dev->hw_desc_pool, chan->oub_ring_virt_addr, 861 chan->oub_phys_addr); 862 } 863 864 static void fsl_re_remove(struct platform_device *ofdev) 865 { 866 struct fsl_re_drv_private *re_priv; 867 struct device *dev; 868 int i; 869 870 dev = &ofdev->dev; 871 re_priv = dev_get_drvdata(dev); 872 873 /* Cleanup chan related memory areas */ 874 for (i = 0; i < re_priv->total_chans; i++) 875 fsl_re_remove_chan(re_priv->re_jrs[i]); 876 877 /* Unregister the driver */ 878 dma_async_device_unregister(&re_priv->dma_dev); 879 } 880 881 static const struct of_device_id fsl_re_ids[] = { 882 { .compatible = "fsl,raideng-v1.0", }, 883 {} 884 }; 885 MODULE_DEVICE_TABLE(of, fsl_re_ids); 886 887 static struct platform_driver fsl_re_driver = { 888 .driver = { 889 .name = "fsl-raideng", 890 .of_match_table = fsl_re_ids, 891 }, 892 .probe = fsl_re_probe, 893 .remove = fsl_re_remove, 894 }; 895 896 module_platform_driver(fsl_re_driver); 897 898 MODULE_AUTHOR("Harninder Rai <harninder.rai@freescale.com>"); 899 MODULE_LICENSE("GPL v2"); 900 MODULE_DESCRIPTION("Freescale RAID Engine Device Driver"); 901