1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Copyright (C) 2006-2009 DENX Software Engineering. 4 * 5 * Author: Yuri Tikhonov <yur@emcraft.com> 6 * 7 * Further porting to arch/powerpc by 8 * Anatolij Gustschin <agust@denx.de> 9 */ 10 11 /* 12 * This driver supports the asynchronous DMA copy and RAID engines available 13 * on the AMCC PPC440SPe Processors. 14 * Based on the Intel Xscale(R) family of I/O Processors (IOP 32x, 33x, 134x) 15 * ADMA driver written by D.Williams. 16 */ 17 18 #include <linux/init.h> 19 #include <linux/module.h> 20 #include <linux/async_tx.h> 21 #include <linux/delay.h> 22 #include <linux/dma-mapping.h> 23 #include <linux/spinlock.h> 24 #include <linux/interrupt.h> 25 #include <linux/slab.h> 26 #include <linux/uaccess.h> 27 #include <linux/proc_fs.h> 28 #include <linux/of.h> 29 #include <linux/of_address.h> 30 #include <linux/of_irq.h> 31 #include <linux/platform_device.h> 32 #include <asm/dcr.h> 33 #include <asm/dcr-regs.h> 34 #include "adma.h" 35 #include "../dmaengine.h" 36 37 enum ppc_adma_init_code { 38 PPC_ADMA_INIT_OK = 0, 39 PPC_ADMA_INIT_MEMRES, 40 PPC_ADMA_INIT_MEMREG, 41 PPC_ADMA_INIT_ALLOC, 42 PPC_ADMA_INIT_COHERENT, 43 PPC_ADMA_INIT_CHANNEL, 44 PPC_ADMA_INIT_IRQ1, 45 PPC_ADMA_INIT_IRQ2, 46 PPC_ADMA_INIT_REGISTER 47 }; 48 49 static char *ppc_adma_errors[] = { 50 [PPC_ADMA_INIT_OK] = "ok", 51 [PPC_ADMA_INIT_MEMRES] = "failed to get memory resource", 52 [PPC_ADMA_INIT_MEMREG] = "failed to request memory region", 53 [PPC_ADMA_INIT_ALLOC] = "failed to allocate memory for adev " 54 "structure", 55 [PPC_ADMA_INIT_COHERENT] = "failed to allocate coherent memory for " 56 "hardware descriptors", 57 [PPC_ADMA_INIT_CHANNEL] = "failed to allocate memory for channel", 58 [PPC_ADMA_INIT_IRQ1] = "failed to request first irq", 59 [PPC_ADMA_INIT_IRQ2] = "failed to request second irq", 60 [PPC_ADMA_INIT_REGISTER] = "failed to register dma async device", 61 }; 62 63 static enum ppc_adma_init_code 64 ppc440spe_adma_devices[PPC440SPE_ADMA_ENGINES_NUM]; 65 66 struct ppc_dma_chan_ref { 67 struct dma_chan *chan; 68 struct list_head node; 69 }; 70 71 /* The list of channels exported by ppc440spe ADMA */ 72 static struct list_head 73 ppc440spe_adma_chan_list = LIST_HEAD_INIT(ppc440spe_adma_chan_list); 74 75 /* This flag is set when want to refetch the xor chain in the interrupt 76 * handler 77 */ 78 static u32 do_xor_refetch; 79 80 /* Pointer to DMA0, DMA1 CP/CS FIFO */ 81 static void *ppc440spe_dma_fifo_buf; 82 83 /* Pointers to last submitted to DMA0, DMA1 CDBs */ 84 static struct ppc440spe_adma_desc_slot *chan_last_sub[3]; 85 static struct ppc440spe_adma_desc_slot *chan_first_cdb[3]; 86 87 /* Pointer to last linked and submitted xor CB */ 88 static struct ppc440spe_adma_desc_slot *xor_last_linked; 89 static struct ppc440spe_adma_desc_slot *xor_last_submit; 90 91 /* This array is used in data-check operations for storing a pattern */ 92 static char ppc440spe_qword[16]; 93 94 static atomic_t ppc440spe_adma_err_irq_ref; 95 static dcr_host_t ppc440spe_mq_dcr_host; 96 static unsigned int ppc440spe_mq_dcr_len; 97 98 /* Since RXOR operations use the common register (MQ0_CF2H) for setting-up 99 * the block size in transactions, then we do not allow to activate more than 100 * only one RXOR transactions simultaneously. So use this var to store 101 * the information about is RXOR currently active (PPC440SPE_RXOR_RUN bit is 102 * set) or not (PPC440SPE_RXOR_RUN is clear). 103 */ 104 static unsigned long ppc440spe_rxor_state; 105 106 /* These are used in enable & check routines 107 */ 108 static u32 ppc440spe_r6_enabled; 109 static struct ppc440spe_adma_chan *ppc440spe_r6_tchan; 110 static struct completion ppc440spe_r6_test_comp; 111 112 static int ppc440spe_adma_dma2rxor_prep_src( 113 struct ppc440spe_adma_desc_slot *desc, 114 struct ppc440spe_rxor *cursor, int index, 115 int src_cnt, u32 addr); 116 static void ppc440spe_adma_dma2rxor_set_src( 117 struct ppc440spe_adma_desc_slot *desc, 118 int index, dma_addr_t addr); 119 static void ppc440spe_adma_dma2rxor_set_mult( 120 struct ppc440spe_adma_desc_slot *desc, 121 int index, u8 mult); 122 123 #ifdef ADMA_LL_DEBUG 124 #define ADMA_LL_DBG(x) ({ if (1) x; 0; }) 125 #else 126 #define ADMA_LL_DBG(x) ({ if (0) x; 0; }) 127 #endif 128 129 static void print_cb(struct ppc440spe_adma_chan *chan, void *block) 130 { 131 struct dma_cdb *cdb; 132 struct xor_cb *cb; 133 int i; 134 135 switch (chan->device->id) { 136 case 0: 137 case 1: 138 cdb = block; 139 140 pr_debug("CDB at %p [%d]:\n" 141 "\t attr 0x%02x opc 0x%02x cnt 0x%08x\n" 142 "\t sg1u 0x%08x sg1l 0x%08x\n" 143 "\t sg2u 0x%08x sg2l 0x%08x\n" 144 "\t sg3u 0x%08x sg3l 0x%08x\n", 145 cdb, chan->device->id, 146 cdb->attr, cdb->opc, le32_to_cpu(cdb->cnt), 147 le32_to_cpu(cdb->sg1u), le32_to_cpu(cdb->sg1l), 148 le32_to_cpu(cdb->sg2u), le32_to_cpu(cdb->sg2l), 149 le32_to_cpu(cdb->sg3u), le32_to_cpu(cdb->sg3l) 150 ); 151 break; 152 case 2: 153 cb = block; 154 155 pr_debug("CB at %p [%d]:\n" 156 "\t cbc 0x%08x cbbc 0x%08x cbs 0x%08x\n" 157 "\t cbtah 0x%08x cbtal 0x%08x\n" 158 "\t cblah 0x%08x cblal 0x%08x\n", 159 cb, chan->device->id, 160 cb->cbc, cb->cbbc, cb->cbs, 161 cb->cbtah, cb->cbtal, 162 cb->cblah, cb->cblal); 163 for (i = 0; i < 16; i++) { 164 if (i && !cb->ops[i].h && !cb->ops[i].l) 165 continue; 166 pr_debug("\t ops[%2d]: h 0x%08x l 0x%08x\n", 167 i, cb->ops[i].h, cb->ops[i].l); 168 } 169 break; 170 } 171 } 172 173 static void print_cb_list(struct ppc440spe_adma_chan *chan, 174 struct ppc440spe_adma_desc_slot *iter) 175 { 176 for (; iter; iter = iter->hw_next) 177 print_cb(chan, iter->hw_desc); 178 } 179 180 static void prep_dma_xor_dbg(int id, dma_addr_t dst, dma_addr_t *src, 181 unsigned int src_cnt) 182 { 183 int i; 184 185 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 186 for (i = 0; i < src_cnt; i++) 187 pr_debug("\t0x%016llx ", src[i]); 188 pr_debug("dst:\n\t0x%016llx\n", dst); 189 } 190 191 static void prep_dma_pq_dbg(int id, dma_addr_t *dst, dma_addr_t *src, 192 unsigned int src_cnt) 193 { 194 int i; 195 196 pr_debug("\n%s(%d):\nsrc: ", __func__, id); 197 for (i = 0; i < src_cnt; i++) 198 pr_debug("\t0x%016llx ", src[i]); 199 pr_debug("dst: "); 200 for (i = 0; i < 2; i++) 201 pr_debug("\t0x%016llx ", dst[i]); 202 } 203 204 static void prep_dma_pqzero_sum_dbg(int id, dma_addr_t *src, 205 unsigned int src_cnt, 206 const unsigned char *scf) 207 { 208 int i; 209 210 pr_debug("\n%s(%d):\nsrc(coef): ", __func__, id); 211 if (scf) { 212 for (i = 0; i < src_cnt; i++) 213 pr_debug("\t0x%016llx(0x%02x) ", src[i], scf[i]); 214 } else { 215 for (i = 0; i < src_cnt; i++) 216 pr_debug("\t0x%016llx(no) ", src[i]); 217 } 218 219 pr_debug("dst: "); 220 for (i = 0; i < 2; i++) 221 pr_debug("\t0x%016llx ", src[src_cnt + i]); 222 } 223 224 /****************************************************************************** 225 * Command (Descriptor) Blocks low-level routines 226 ******************************************************************************/ 227 /** 228 * ppc440spe_desc_init_interrupt - initialize the descriptor for INTERRUPT 229 * pseudo operation 230 */ 231 static void ppc440spe_desc_init_interrupt(struct ppc440spe_adma_desc_slot *desc, 232 struct ppc440spe_adma_chan *chan) 233 { 234 struct xor_cb *p; 235 236 switch (chan->device->id) { 237 case PPC440SPE_XOR_ID: 238 p = desc->hw_desc; 239 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 240 /* NOP with Command Block Complete Enable */ 241 p->cbc = XOR_CBCR_CBCE_BIT; 242 break; 243 case PPC440SPE_DMA0_ID: 244 case PPC440SPE_DMA1_ID: 245 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 246 /* NOP with interrupt */ 247 set_bit(PPC440SPE_DESC_INT, &desc->flags); 248 break; 249 default: 250 printk(KERN_ERR "Unsupported id %d in %s\n", chan->device->id, 251 __func__); 252 break; 253 } 254 } 255 256 /** 257 * ppc440spe_desc_init_null_xor - initialize the descriptor for NULL XOR 258 * pseudo operation 259 */ 260 static void ppc440spe_desc_init_null_xor(struct ppc440spe_adma_desc_slot *desc) 261 { 262 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 263 desc->hw_next = NULL; 264 desc->src_cnt = 0; 265 desc->dst_cnt = 1; 266 } 267 268 /** 269 * ppc440spe_desc_init_xor - initialize the descriptor for XOR operation 270 */ 271 static void ppc440spe_desc_init_xor(struct ppc440spe_adma_desc_slot *desc, 272 int src_cnt, unsigned long flags) 273 { 274 struct xor_cb *hw_desc = desc->hw_desc; 275 276 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 277 desc->hw_next = NULL; 278 desc->src_cnt = src_cnt; 279 desc->dst_cnt = 1; 280 281 hw_desc->cbc = XOR_CBCR_TGT_BIT | src_cnt; 282 if (flags & DMA_PREP_INTERRUPT) 283 /* Enable interrupt on completion */ 284 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 285 } 286 287 /** 288 * ppc440spe_desc_init_dma2pq - initialize the descriptor for PQ 289 * operation in DMA2 controller 290 */ 291 static void ppc440spe_desc_init_dma2pq(struct ppc440spe_adma_desc_slot *desc, 292 int dst_cnt, int src_cnt, unsigned long flags) 293 { 294 struct xor_cb *hw_desc = desc->hw_desc; 295 296 memset(desc->hw_desc, 0, sizeof(struct xor_cb)); 297 desc->hw_next = NULL; 298 desc->src_cnt = src_cnt; 299 desc->dst_cnt = dst_cnt; 300 memset(desc->reverse_flags, 0, sizeof(desc->reverse_flags)); 301 desc->descs_per_op = 0; 302 303 hw_desc->cbc = XOR_CBCR_TGT_BIT; 304 if (flags & DMA_PREP_INTERRUPT) 305 /* Enable interrupt on completion */ 306 hw_desc->cbc |= XOR_CBCR_CBCE_BIT; 307 } 308 309 #define DMA_CTRL_FLAGS_LAST DMA_PREP_FENCE 310 #define DMA_PREP_ZERO_P (DMA_CTRL_FLAGS_LAST << 1) 311 #define DMA_PREP_ZERO_Q (DMA_PREP_ZERO_P << 1) 312 313 /** 314 * ppc440spe_desc_init_dma01pq - initialize the descriptors for PQ operation 315 * with DMA0/1 316 */ 317 static void ppc440spe_desc_init_dma01pq(struct ppc440spe_adma_desc_slot *desc, 318 int dst_cnt, int src_cnt, unsigned long flags, 319 unsigned long op) 320 { 321 struct dma_cdb *hw_desc; 322 struct ppc440spe_adma_desc_slot *iter; 323 u8 dopc; 324 325 /* Common initialization of a PQ descriptors chain */ 326 set_bits(op, &desc->flags); 327 desc->src_cnt = src_cnt; 328 desc->dst_cnt = dst_cnt; 329 330 /* WXOR MULTICAST if both P and Q are being computed 331 * MV_SG1_SG2 if Q only 332 */ 333 dopc = (desc->dst_cnt == DMA_DEST_MAX_NUM) ? 334 DMA_CDB_OPC_MULTICAST : DMA_CDB_OPC_MV_SG1_SG2; 335 336 list_for_each_entry(iter, &desc->group_list, chain_node) { 337 hw_desc = iter->hw_desc; 338 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 339 340 if (likely(!list_is_last(&iter->chain_node, 341 &desc->group_list))) { 342 /* set 'next' pointer */ 343 iter->hw_next = list_entry(iter->chain_node.next, 344 struct ppc440spe_adma_desc_slot, chain_node); 345 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 346 } else { 347 /* this is the last descriptor. 348 * this slot will be pasted from ADMA level 349 * each time it wants to configure parameters 350 * of the transaction (src, dst, ...) 351 */ 352 iter->hw_next = NULL; 353 if (flags & DMA_PREP_INTERRUPT) 354 set_bit(PPC440SPE_DESC_INT, &iter->flags); 355 else 356 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 357 } 358 } 359 360 /* Set OPS depending on WXOR/RXOR type of operation */ 361 if (!test_bit(PPC440SPE_DESC_RXOR, &desc->flags)) { 362 /* This is a WXOR only chain: 363 * - first descriptors are for zeroing destinations 364 * if PPC440SPE_ZERO_P/Q set; 365 * - descriptors remained are for GF-XOR operations. 366 */ 367 iter = list_first_entry(&desc->group_list, 368 struct ppc440spe_adma_desc_slot, 369 chain_node); 370 371 if (test_bit(PPC440SPE_ZERO_P, &desc->flags)) { 372 hw_desc = iter->hw_desc; 373 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 374 iter = list_first_entry(&iter->chain_node, 375 struct ppc440spe_adma_desc_slot, 376 chain_node); 377 } 378 379 if (test_bit(PPC440SPE_ZERO_Q, &desc->flags)) { 380 hw_desc = iter->hw_desc; 381 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 382 iter = list_first_entry(&iter->chain_node, 383 struct ppc440spe_adma_desc_slot, 384 chain_node); 385 } 386 387 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 388 hw_desc = iter->hw_desc; 389 hw_desc->opc = dopc; 390 } 391 } else { 392 /* This is either RXOR-only or mixed RXOR/WXOR */ 393 394 /* The first 1 or 2 slots in chain are always RXOR, 395 * if need to calculate P & Q, then there are two 396 * RXOR slots; if only P or only Q, then there is one 397 */ 398 iter = list_first_entry(&desc->group_list, 399 struct ppc440spe_adma_desc_slot, 400 chain_node); 401 hw_desc = iter->hw_desc; 402 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 403 404 if (desc->dst_cnt == DMA_DEST_MAX_NUM) { 405 iter = list_first_entry(&iter->chain_node, 406 struct ppc440spe_adma_desc_slot, 407 chain_node); 408 hw_desc = iter->hw_desc; 409 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 410 } 411 412 /* The remaining descs (if any) are WXORs */ 413 if (test_bit(PPC440SPE_DESC_WXOR, &desc->flags)) { 414 iter = list_first_entry(&iter->chain_node, 415 struct ppc440spe_adma_desc_slot, 416 chain_node); 417 list_for_each_entry_from(iter, &desc->group_list, 418 chain_node) { 419 hw_desc = iter->hw_desc; 420 hw_desc->opc = dopc; 421 } 422 } 423 } 424 } 425 426 /** 427 * ppc440spe_desc_init_dma01pqzero_sum - initialize the descriptor 428 * for PQ_ZERO_SUM operation 429 */ 430 static void ppc440spe_desc_init_dma01pqzero_sum( 431 struct ppc440spe_adma_desc_slot *desc, 432 int dst_cnt, int src_cnt) 433 { 434 struct dma_cdb *hw_desc; 435 struct ppc440spe_adma_desc_slot *iter; 436 int i = 0; 437 u8 dopc = (dst_cnt == 2) ? DMA_CDB_OPC_MULTICAST : 438 DMA_CDB_OPC_MV_SG1_SG2; 439 /* 440 * Initialize starting from 2nd or 3rd descriptor dependent 441 * on dst_cnt. First one or two slots are for cloning P 442 * and/or Q to chan->pdest and/or chan->qdest as we have 443 * to preserve original P/Q. 444 */ 445 iter = list_first_entry(&desc->group_list, 446 struct ppc440spe_adma_desc_slot, chain_node); 447 iter = list_entry(iter->chain_node.next, 448 struct ppc440spe_adma_desc_slot, chain_node); 449 450 if (dst_cnt > 1) { 451 iter = list_entry(iter->chain_node.next, 452 struct ppc440spe_adma_desc_slot, chain_node); 453 } 454 /* initialize each source descriptor in chain */ 455 list_for_each_entry_from(iter, &desc->group_list, chain_node) { 456 hw_desc = iter->hw_desc; 457 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 458 iter->src_cnt = 0; 459 iter->dst_cnt = 0; 460 461 /* This is a ZERO_SUM operation: 462 * - <src_cnt> descriptors starting from 2nd or 3rd 463 * descriptor are for GF-XOR operations; 464 * - remaining <dst_cnt> descriptors are for checking the result 465 */ 466 if (i++ < src_cnt) 467 /* MV_SG1_SG2 if only Q is being verified 468 * MULTICAST if both P and Q are being verified 469 */ 470 hw_desc->opc = dopc; 471 else 472 /* DMA_CDB_OPC_DCHECK128 operation */ 473 hw_desc->opc = DMA_CDB_OPC_DCHECK128; 474 475 if (likely(!list_is_last(&iter->chain_node, 476 &desc->group_list))) { 477 /* set 'next' pointer */ 478 iter->hw_next = list_entry(iter->chain_node.next, 479 struct ppc440spe_adma_desc_slot, 480 chain_node); 481 } else { 482 /* this is the last descriptor. 483 * this slot will be pasted from ADMA level 484 * each time it wants to configure parameters 485 * of the transaction (src, dst, ...) 486 */ 487 iter->hw_next = NULL; 488 /* always enable interrupt generation since we get 489 * the status of pqzero from the handler 490 */ 491 set_bit(PPC440SPE_DESC_INT, &iter->flags); 492 } 493 } 494 desc->src_cnt = src_cnt; 495 desc->dst_cnt = dst_cnt; 496 } 497 498 /** 499 * ppc440spe_desc_init_memcpy - initialize the descriptor for MEMCPY operation 500 */ 501 static void ppc440spe_desc_init_memcpy(struct ppc440spe_adma_desc_slot *desc, 502 unsigned long flags) 503 { 504 struct dma_cdb *hw_desc = desc->hw_desc; 505 506 memset(desc->hw_desc, 0, sizeof(struct dma_cdb)); 507 desc->hw_next = NULL; 508 desc->src_cnt = 1; 509 desc->dst_cnt = 1; 510 511 if (flags & DMA_PREP_INTERRUPT) 512 set_bit(PPC440SPE_DESC_INT, &desc->flags); 513 else 514 clear_bit(PPC440SPE_DESC_INT, &desc->flags); 515 516 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 517 } 518 519 /** 520 * ppc440spe_desc_set_src_addr - set source address into the descriptor 521 */ 522 static void ppc440spe_desc_set_src_addr(struct ppc440spe_adma_desc_slot *desc, 523 struct ppc440spe_adma_chan *chan, 524 int src_idx, dma_addr_t addrh, 525 dma_addr_t addrl) 526 { 527 struct dma_cdb *dma_hw_desc; 528 struct xor_cb *xor_hw_desc; 529 phys_addr_t addr64, tmplow, tmphi; 530 531 switch (chan->device->id) { 532 case PPC440SPE_DMA0_ID: 533 case PPC440SPE_DMA1_ID: 534 if (!addrh) { 535 addr64 = addrl; 536 tmphi = (addr64 >> 32); 537 tmplow = (addr64 & 0xFFFFFFFF); 538 } else { 539 tmphi = addrh; 540 tmplow = addrl; 541 } 542 dma_hw_desc = desc->hw_desc; 543 dma_hw_desc->sg1l = cpu_to_le32((u32)tmplow); 544 dma_hw_desc->sg1u |= cpu_to_le32((u32)tmphi); 545 break; 546 case PPC440SPE_XOR_ID: 547 xor_hw_desc = desc->hw_desc; 548 xor_hw_desc->ops[src_idx].l = addrl; 549 xor_hw_desc->ops[src_idx].h |= addrh; 550 break; 551 } 552 } 553 554 /** 555 * ppc440spe_desc_set_src_mult - set source address mult into the descriptor 556 */ 557 static void ppc440spe_desc_set_src_mult(struct ppc440spe_adma_desc_slot *desc, 558 struct ppc440spe_adma_chan *chan, u32 mult_index, 559 int sg_index, unsigned char mult_value) 560 { 561 struct dma_cdb *dma_hw_desc; 562 u32 *psgu; 563 564 switch (chan->device->id) { 565 case PPC440SPE_DMA0_ID: 566 case PPC440SPE_DMA1_ID: 567 dma_hw_desc = desc->hw_desc; 568 569 switch (sg_index) { 570 /* for RXOR operations set multiplier 571 * into source cued address 572 */ 573 case DMA_CDB_SG_SRC: 574 psgu = &dma_hw_desc->sg1u; 575 break; 576 /* for WXOR operations set multiplier 577 * into destination cued address(es) 578 */ 579 case DMA_CDB_SG_DST1: 580 psgu = &dma_hw_desc->sg2u; 581 break; 582 case DMA_CDB_SG_DST2: 583 psgu = &dma_hw_desc->sg3u; 584 break; 585 default: 586 BUG(); 587 } 588 589 *psgu |= cpu_to_le32(mult_value << mult_index); 590 break; 591 case PPC440SPE_XOR_ID: 592 break; 593 default: 594 BUG(); 595 } 596 } 597 598 /** 599 * ppc440spe_desc_set_dest_addr - set destination address into the descriptor 600 */ 601 static void ppc440spe_desc_set_dest_addr(struct ppc440spe_adma_desc_slot *desc, 602 struct ppc440spe_adma_chan *chan, 603 dma_addr_t addrh, dma_addr_t addrl, 604 u32 dst_idx) 605 { 606 struct dma_cdb *dma_hw_desc; 607 struct xor_cb *xor_hw_desc; 608 phys_addr_t addr64, tmphi, tmplow; 609 u32 *psgu, *psgl; 610 611 switch (chan->device->id) { 612 case PPC440SPE_DMA0_ID: 613 case PPC440SPE_DMA1_ID: 614 if (!addrh) { 615 addr64 = addrl; 616 tmphi = (addr64 >> 32); 617 tmplow = (addr64 & 0xFFFFFFFF); 618 } else { 619 tmphi = addrh; 620 tmplow = addrl; 621 } 622 dma_hw_desc = desc->hw_desc; 623 624 psgu = dst_idx ? &dma_hw_desc->sg3u : &dma_hw_desc->sg2u; 625 psgl = dst_idx ? &dma_hw_desc->sg3l : &dma_hw_desc->sg2l; 626 627 *psgl = cpu_to_le32((u32)tmplow); 628 *psgu |= cpu_to_le32((u32)tmphi); 629 break; 630 case PPC440SPE_XOR_ID: 631 xor_hw_desc = desc->hw_desc; 632 xor_hw_desc->cbtal = addrl; 633 xor_hw_desc->cbtah |= addrh; 634 break; 635 } 636 } 637 638 /** 639 * ppc440spe_desc_set_byte_count - set number of data bytes involved 640 * into the operation 641 */ 642 static void ppc440spe_desc_set_byte_count(struct ppc440spe_adma_desc_slot *desc, 643 struct ppc440spe_adma_chan *chan, 644 u32 byte_count) 645 { 646 struct dma_cdb *dma_hw_desc; 647 struct xor_cb *xor_hw_desc; 648 649 switch (chan->device->id) { 650 case PPC440SPE_DMA0_ID: 651 case PPC440SPE_DMA1_ID: 652 dma_hw_desc = desc->hw_desc; 653 dma_hw_desc->cnt = cpu_to_le32(byte_count); 654 break; 655 case PPC440SPE_XOR_ID: 656 xor_hw_desc = desc->hw_desc; 657 xor_hw_desc->cbbc = byte_count; 658 break; 659 } 660 } 661 662 /** 663 * ppc440spe_desc_set_rxor_block_size - set RXOR block size 664 */ 665 static inline void ppc440spe_desc_set_rxor_block_size(u32 byte_count) 666 { 667 /* assume that byte_count is aligned on the 512-boundary; 668 * thus write it directly to the register (bits 23:31 are 669 * reserved there). 670 */ 671 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CF2H, byte_count); 672 } 673 674 /** 675 * ppc440spe_desc_set_dcheck - set CHECK pattern 676 */ 677 static void ppc440spe_desc_set_dcheck(struct ppc440spe_adma_desc_slot *desc, 678 struct ppc440spe_adma_chan *chan, u8 *qword) 679 { 680 struct dma_cdb *dma_hw_desc; 681 682 switch (chan->device->id) { 683 case PPC440SPE_DMA0_ID: 684 case PPC440SPE_DMA1_ID: 685 dma_hw_desc = desc->hw_desc; 686 iowrite32(qword[0], &dma_hw_desc->sg3l); 687 iowrite32(qword[4], &dma_hw_desc->sg3u); 688 iowrite32(qword[8], &dma_hw_desc->sg2l); 689 iowrite32(qword[12], &dma_hw_desc->sg2u); 690 break; 691 default: 692 BUG(); 693 } 694 } 695 696 /** 697 * ppc440spe_xor_set_link - set link address in xor CB 698 */ 699 static void ppc440spe_xor_set_link(struct ppc440spe_adma_desc_slot *prev_desc, 700 struct ppc440spe_adma_desc_slot *next_desc) 701 { 702 struct xor_cb *xor_hw_desc = prev_desc->hw_desc; 703 704 if (unlikely(!next_desc || !(next_desc->phys))) { 705 printk(KERN_ERR "%s: next_desc=0x%p; next_desc->phys=0x%llx\n", 706 __func__, next_desc, 707 next_desc ? next_desc->phys : 0); 708 BUG(); 709 } 710 711 xor_hw_desc->cbs = 0; 712 xor_hw_desc->cblal = next_desc->phys; 713 xor_hw_desc->cblah = 0; 714 xor_hw_desc->cbc |= XOR_CBCR_LNK_BIT; 715 } 716 717 /** 718 * ppc440spe_desc_set_link - set the address of descriptor following this 719 * descriptor in chain 720 */ 721 static void ppc440spe_desc_set_link(struct ppc440spe_adma_chan *chan, 722 struct ppc440spe_adma_desc_slot *prev_desc, 723 struct ppc440spe_adma_desc_slot *next_desc) 724 { 725 unsigned long flags; 726 struct ppc440spe_adma_desc_slot *tail = next_desc; 727 728 if (unlikely(!prev_desc || !next_desc || 729 (prev_desc->hw_next && prev_desc->hw_next != next_desc))) { 730 /* If previous next is overwritten something is wrong. 731 * though we may refetch from append to initiate list 732 * processing; in this case - it's ok. 733 */ 734 printk(KERN_ERR "%s: prev_desc=0x%p; next_desc=0x%p; " 735 "prev->hw_next=0x%p\n", __func__, prev_desc, 736 next_desc, prev_desc ? prev_desc->hw_next : 0); 737 BUG(); 738 } 739 740 local_irq_save(flags); 741 742 /* do s/w chaining both for DMA and XOR descriptors */ 743 prev_desc->hw_next = next_desc; 744 745 switch (chan->device->id) { 746 case PPC440SPE_DMA0_ID: 747 case PPC440SPE_DMA1_ID: 748 break; 749 case PPC440SPE_XOR_ID: 750 /* bind descriptor to the chain */ 751 while (tail->hw_next) 752 tail = tail->hw_next; 753 xor_last_linked = tail; 754 755 if (prev_desc == xor_last_submit) 756 /* do not link to the last submitted CB */ 757 break; 758 ppc440spe_xor_set_link(prev_desc, next_desc); 759 break; 760 } 761 762 local_irq_restore(flags); 763 } 764 765 /** 766 * ppc440spe_desc_get_link - get the address of the descriptor that 767 * follows this one 768 */ 769 static inline u32 ppc440spe_desc_get_link(struct ppc440spe_adma_desc_slot *desc, 770 struct ppc440spe_adma_chan *chan) 771 { 772 if (!desc->hw_next) 773 return 0; 774 775 return desc->hw_next->phys; 776 } 777 778 /** 779 * ppc440spe_desc_is_aligned - check alignment 780 */ 781 static inline int ppc440spe_desc_is_aligned( 782 struct ppc440spe_adma_desc_slot *desc, int num_slots) 783 { 784 return (desc->idx & (num_slots - 1)) ? 0 : 1; 785 } 786 787 /** 788 * ppc440spe_chan_xor_slot_count - get the number of slots necessary for 789 * XOR operation 790 */ 791 static int ppc440spe_chan_xor_slot_count(size_t len, int src_cnt, 792 int *slots_per_op) 793 { 794 int slot_cnt; 795 796 /* each XOR descriptor provides up to 16 source operands */ 797 slot_cnt = *slots_per_op = (src_cnt + XOR_MAX_OPS - 1)/XOR_MAX_OPS; 798 799 if (likely(len <= PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT)) 800 return slot_cnt; 801 802 printk(KERN_ERR "%s: len %d > max %d !!\n", 803 __func__, len, PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 804 BUG(); 805 return slot_cnt; 806 } 807 808 /** 809 * ppc440spe_dma2_pq_slot_count - get the number of slots necessary for 810 * DMA2 PQ operation 811 */ 812 static int ppc440spe_dma2_pq_slot_count(dma_addr_t *srcs, 813 int src_cnt, size_t len) 814 { 815 signed long long order = 0; 816 int state = 0; 817 int addr_count = 0; 818 int i; 819 for (i = 1; i < src_cnt; i++) { 820 dma_addr_t cur_addr = srcs[i]; 821 dma_addr_t old_addr = srcs[i-1]; 822 switch (state) { 823 case 0: 824 if (cur_addr == old_addr + len) { 825 /* direct RXOR */ 826 order = 1; 827 state = 1; 828 if (i == src_cnt-1) 829 addr_count++; 830 } else if (old_addr == cur_addr + len) { 831 /* reverse RXOR */ 832 order = -1; 833 state = 1; 834 if (i == src_cnt-1) 835 addr_count++; 836 } else { 837 state = 3; 838 } 839 break; 840 case 1: 841 if (i == src_cnt-2 || (order == -1 842 && cur_addr != old_addr - len)) { 843 order = 0; 844 state = 0; 845 addr_count++; 846 } else if (cur_addr == old_addr + len*order) { 847 state = 2; 848 if (i == src_cnt-1) 849 addr_count++; 850 } else if (cur_addr == old_addr + 2*len) { 851 state = 2; 852 if (i == src_cnt-1) 853 addr_count++; 854 } else if (cur_addr == old_addr + 3*len) { 855 state = 2; 856 if (i == src_cnt-1) 857 addr_count++; 858 } else { 859 order = 0; 860 state = 0; 861 addr_count++; 862 } 863 break; 864 case 2: 865 order = 0; 866 state = 0; 867 addr_count++; 868 break; 869 } 870 if (state == 3) 871 break; 872 } 873 if (src_cnt <= 1 || (state != 1 && state != 2)) { 874 pr_err("%s: src_cnt=%d, state=%d, addr_count=%d, order=%lld\n", 875 __func__, src_cnt, state, addr_count, order); 876 for (i = 0; i < src_cnt; i++) 877 pr_err("\t[%d] 0x%llx\n", i, srcs[i]); 878 BUG(); 879 } 880 881 return (addr_count + XOR_MAX_OPS - 1) / XOR_MAX_OPS; 882 } 883 884 885 /****************************************************************************** 886 * ADMA channel low-level routines 887 ******************************************************************************/ 888 889 static u32 890 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan); 891 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan); 892 893 /** 894 * ppc440spe_adma_device_clear_eot_status - interrupt ack to XOR or DMA engine 895 */ 896 static void ppc440spe_adma_device_clear_eot_status( 897 struct ppc440spe_adma_chan *chan) 898 { 899 struct dma_regs *dma_reg; 900 struct xor_regs *xor_reg; 901 u8 *p = chan->device->dma_desc_pool_virt; 902 struct dma_cdb *cdb; 903 u32 rv, i; 904 905 switch (chan->device->id) { 906 case PPC440SPE_DMA0_ID: 907 case PPC440SPE_DMA1_ID: 908 /* read FIFO to ack */ 909 dma_reg = chan->device->dma_reg; 910 while ((rv = ioread32(&dma_reg->csfpl))) { 911 i = rv & DMA_CDB_ADDR_MSK; 912 cdb = (struct dma_cdb *)&p[i - 913 (u32)chan->device->dma_desc_pool]; 914 915 /* Clear opcode to ack. This is necessary for 916 * ZeroSum operations only 917 */ 918 cdb->opc = 0; 919 920 if (test_bit(PPC440SPE_RXOR_RUN, 921 &ppc440spe_rxor_state)) { 922 /* probably this is a completed RXOR op, 923 * get pointer to CDB using the fact that 924 * physical and virtual addresses of CDB 925 * in pools have the same offsets 926 */ 927 if (le32_to_cpu(cdb->sg1u) & 928 DMA_CUED_XOR_BASE) { 929 /* this is a RXOR */ 930 clear_bit(PPC440SPE_RXOR_RUN, 931 &ppc440spe_rxor_state); 932 } 933 } 934 935 if (rv & DMA_CDB_STATUS_MSK) { 936 /* ZeroSum check failed 937 */ 938 struct ppc440spe_adma_desc_slot *iter; 939 dma_addr_t phys = rv & ~DMA_CDB_MSK; 940 941 /* 942 * Update the status of corresponding 943 * descriptor. 944 */ 945 list_for_each_entry(iter, &chan->chain, 946 chain_node) { 947 if (iter->phys == phys) 948 break; 949 } 950 /* 951 * if cannot find the corresponding 952 * slot it's a bug 953 */ 954 BUG_ON(&iter->chain_node == &chan->chain); 955 956 if (iter->xor_check_result) { 957 if (test_bit(PPC440SPE_DESC_PCHECK, 958 &iter->flags)) { 959 *iter->xor_check_result |= 960 SUM_CHECK_P_RESULT; 961 } else 962 if (test_bit(PPC440SPE_DESC_QCHECK, 963 &iter->flags)) { 964 *iter->xor_check_result |= 965 SUM_CHECK_Q_RESULT; 966 } else 967 BUG(); 968 } 969 } 970 } 971 972 rv = ioread32(&dma_reg->dsts); 973 if (rv) { 974 pr_err("DMA%d err status: 0x%x\n", 975 chan->device->id, rv); 976 /* write back to clear */ 977 iowrite32(rv, &dma_reg->dsts); 978 } 979 break; 980 case PPC440SPE_XOR_ID: 981 /* reset status bits to ack */ 982 xor_reg = chan->device->xor_reg; 983 rv = ioread32be(&xor_reg->sr); 984 iowrite32be(rv, &xor_reg->sr); 985 986 if (rv & (XOR_IE_ICBIE_BIT|XOR_IE_ICIE_BIT|XOR_IE_RPTIE_BIT)) { 987 if (rv & XOR_IE_RPTIE_BIT) { 988 /* Read PLB Timeout Error. 989 * Try to resubmit the CB 990 */ 991 u32 val = ioread32be(&xor_reg->ccbalr); 992 993 iowrite32be(val, &xor_reg->cblalr); 994 995 val = ioread32be(&xor_reg->crsr); 996 iowrite32be(val | XOR_CRSR_XAE_BIT, 997 &xor_reg->crsr); 998 } else 999 pr_err("XOR ERR 0x%x status\n", rv); 1000 break; 1001 } 1002 1003 /* if the XORcore is idle, but there are unprocessed CBs 1004 * then refetch the s/w chain here 1005 */ 1006 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) && 1007 do_xor_refetch) 1008 ppc440spe_chan_append(chan); 1009 break; 1010 } 1011 } 1012 1013 /** 1014 * ppc440spe_chan_is_busy - get the channel status 1015 */ 1016 static int ppc440spe_chan_is_busy(struct ppc440spe_adma_chan *chan) 1017 { 1018 struct dma_regs *dma_reg; 1019 struct xor_regs *xor_reg; 1020 int busy = 0; 1021 1022 switch (chan->device->id) { 1023 case PPC440SPE_DMA0_ID: 1024 case PPC440SPE_DMA1_ID: 1025 dma_reg = chan->device->dma_reg; 1026 /* if command FIFO's head and tail pointers are equal and 1027 * status tail is the same as command, then channel is free 1028 */ 1029 if (ioread16(&dma_reg->cpfhp) != ioread16(&dma_reg->cpftp) || 1030 ioread16(&dma_reg->cpftp) != ioread16(&dma_reg->csftp)) 1031 busy = 1; 1032 break; 1033 case PPC440SPE_XOR_ID: 1034 /* use the special status bit for the XORcore 1035 */ 1036 xor_reg = chan->device->xor_reg; 1037 busy = (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) ? 1 : 0; 1038 break; 1039 } 1040 1041 return busy; 1042 } 1043 1044 /** 1045 * ppc440spe_chan_set_first_xor_descriptor - init XORcore chain 1046 */ 1047 static void ppc440spe_chan_set_first_xor_descriptor( 1048 struct ppc440spe_adma_chan *chan, 1049 struct ppc440spe_adma_desc_slot *next_desc) 1050 { 1051 struct xor_regs *xor_reg = chan->device->xor_reg; 1052 1053 if (ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT) 1054 printk(KERN_INFO "%s: Warn: XORcore is running " 1055 "when try to set the first CDB!\n", 1056 __func__); 1057 1058 xor_last_submit = xor_last_linked = next_desc; 1059 1060 iowrite32be(XOR_CRSR_64BA_BIT, &xor_reg->crsr); 1061 1062 iowrite32be(next_desc->phys, &xor_reg->cblalr); 1063 iowrite32be(0, &xor_reg->cblahr); 1064 iowrite32be(ioread32be(&xor_reg->cbcr) | XOR_CBCR_LNK_BIT, 1065 &xor_reg->cbcr); 1066 1067 chan->hw_chain_inited = 1; 1068 } 1069 1070 /** 1071 * ppc440spe_dma_put_desc - put DMA0,1 descriptor to FIFO. 1072 * called with irqs disabled 1073 */ 1074 static void ppc440spe_dma_put_desc(struct ppc440spe_adma_chan *chan, 1075 struct ppc440spe_adma_desc_slot *desc) 1076 { 1077 u32 pcdb; 1078 struct dma_regs *dma_reg = chan->device->dma_reg; 1079 1080 pcdb = desc->phys; 1081 if (!test_bit(PPC440SPE_DESC_INT, &desc->flags)) 1082 pcdb |= DMA_CDB_NO_INT; 1083 1084 chan_last_sub[chan->device->id] = desc; 1085 1086 ADMA_LL_DBG(print_cb(chan, desc->hw_desc)); 1087 1088 iowrite32(pcdb, &dma_reg->cpfpl); 1089 } 1090 1091 /** 1092 * ppc440spe_chan_append - update the h/w chain in the channel 1093 */ 1094 static void ppc440spe_chan_append(struct ppc440spe_adma_chan *chan) 1095 { 1096 struct xor_regs *xor_reg; 1097 struct ppc440spe_adma_desc_slot *iter; 1098 struct xor_cb *xcb; 1099 u32 cur_desc; 1100 unsigned long flags; 1101 1102 local_irq_save(flags); 1103 1104 switch (chan->device->id) { 1105 case PPC440SPE_DMA0_ID: 1106 case PPC440SPE_DMA1_ID: 1107 cur_desc = ppc440spe_chan_get_current_descriptor(chan); 1108 1109 if (likely(cur_desc)) { 1110 iter = chan_last_sub[chan->device->id]; 1111 BUG_ON(!iter); 1112 } else { 1113 /* first peer */ 1114 iter = chan_first_cdb[chan->device->id]; 1115 BUG_ON(!iter); 1116 ppc440spe_dma_put_desc(chan, iter); 1117 chan->hw_chain_inited = 1; 1118 } 1119 1120 /* is there something new to append */ 1121 if (!iter->hw_next) 1122 break; 1123 1124 /* flush descriptors from the s/w queue to fifo */ 1125 list_for_each_entry_continue(iter, &chan->chain, chain_node) { 1126 ppc440spe_dma_put_desc(chan, iter); 1127 if (!iter->hw_next) 1128 break; 1129 } 1130 break; 1131 case PPC440SPE_XOR_ID: 1132 /* update h/w links and refetch */ 1133 if (!xor_last_submit->hw_next) 1134 break; 1135 1136 xor_reg = chan->device->xor_reg; 1137 /* the last linked CDB has to generate an interrupt 1138 * that we'd be able to append the next lists to h/w 1139 * regardless of the XOR engine state at the moment of 1140 * appending of these next lists 1141 */ 1142 xcb = xor_last_linked->hw_desc; 1143 xcb->cbc |= XOR_CBCR_CBCE_BIT; 1144 1145 if (!(ioread32be(&xor_reg->sr) & XOR_SR_XCP_BIT)) { 1146 /* XORcore is idle. Refetch now */ 1147 do_xor_refetch = 0; 1148 ppc440spe_xor_set_link(xor_last_submit, 1149 xor_last_submit->hw_next); 1150 1151 ADMA_LL_DBG(print_cb_list(chan, 1152 xor_last_submit->hw_next)); 1153 1154 xor_last_submit = xor_last_linked; 1155 iowrite32be(ioread32be(&xor_reg->crsr) | 1156 XOR_CRSR_RCBE_BIT | XOR_CRSR_64BA_BIT, 1157 &xor_reg->crsr); 1158 } else { 1159 /* XORcore is running. Refetch later in the handler */ 1160 do_xor_refetch = 1; 1161 } 1162 1163 break; 1164 } 1165 1166 local_irq_restore(flags); 1167 } 1168 1169 /** 1170 * ppc440spe_chan_get_current_descriptor - get the currently executed descriptor 1171 */ 1172 static u32 1173 ppc440spe_chan_get_current_descriptor(struct ppc440spe_adma_chan *chan) 1174 { 1175 struct dma_regs *dma_reg; 1176 struct xor_regs *xor_reg; 1177 1178 if (unlikely(!chan->hw_chain_inited)) 1179 /* h/w descriptor chain is not initialized yet */ 1180 return 0; 1181 1182 switch (chan->device->id) { 1183 case PPC440SPE_DMA0_ID: 1184 case PPC440SPE_DMA1_ID: 1185 dma_reg = chan->device->dma_reg; 1186 return ioread32(&dma_reg->acpl) & (~DMA_CDB_MSK); 1187 case PPC440SPE_XOR_ID: 1188 xor_reg = chan->device->xor_reg; 1189 return ioread32be(&xor_reg->ccbalr); 1190 } 1191 return 0; 1192 } 1193 1194 /** 1195 * ppc440spe_chan_run - enable the channel 1196 */ 1197 static void ppc440spe_chan_run(struct ppc440spe_adma_chan *chan) 1198 { 1199 struct xor_regs *xor_reg; 1200 1201 switch (chan->device->id) { 1202 case PPC440SPE_DMA0_ID: 1203 case PPC440SPE_DMA1_ID: 1204 /* DMAs are always enabled, do nothing */ 1205 break; 1206 case PPC440SPE_XOR_ID: 1207 /* drain write buffer */ 1208 xor_reg = chan->device->xor_reg; 1209 1210 /* fetch descriptor pointed to in <link> */ 1211 iowrite32be(XOR_CRSR_64BA_BIT | XOR_CRSR_XAE_BIT, 1212 &xor_reg->crsr); 1213 break; 1214 } 1215 } 1216 1217 /****************************************************************************** 1218 * ADMA device level 1219 ******************************************************************************/ 1220 1221 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan); 1222 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan); 1223 1224 static dma_cookie_t 1225 ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx); 1226 1227 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *tx, 1228 dma_addr_t addr, int index); 1229 static void 1230 ppc440spe_adma_memcpy_xor_set_src(struct ppc440spe_adma_desc_slot *tx, 1231 dma_addr_t addr, int index); 1232 1233 static void 1234 ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *tx, 1235 dma_addr_t *paddr, unsigned long flags); 1236 static void 1237 ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *tx, 1238 dma_addr_t addr, int index); 1239 static void 1240 ppc440spe_adma_pq_set_src_mult(struct ppc440spe_adma_desc_slot *tx, 1241 unsigned char mult, int index, int dst_pos); 1242 static void 1243 ppc440spe_adma_pqzero_sum_set_dest(struct ppc440spe_adma_desc_slot *tx, 1244 dma_addr_t paddr, dma_addr_t qaddr); 1245 1246 static struct page *ppc440spe_rxor_srcs[32]; 1247 1248 /** 1249 * ppc440spe_can_rxor - check if the operands may be processed with RXOR 1250 */ 1251 static int ppc440spe_can_rxor(struct page **srcs, int src_cnt, size_t len) 1252 { 1253 int i, order = 0, state = 0; 1254 int idx = 0; 1255 1256 if (unlikely(!(src_cnt > 1))) 1257 return 0; 1258 1259 BUG_ON(src_cnt > ARRAY_SIZE(ppc440spe_rxor_srcs)); 1260 1261 /* Skip holes in the source list before checking */ 1262 for (i = 0; i < src_cnt; i++) { 1263 if (!srcs[i]) 1264 continue; 1265 ppc440spe_rxor_srcs[idx++] = srcs[i]; 1266 } 1267 src_cnt = idx; 1268 1269 for (i = 1; i < src_cnt; i++) { 1270 char *cur_addr = page_address(ppc440spe_rxor_srcs[i]); 1271 char *old_addr = page_address(ppc440spe_rxor_srcs[i - 1]); 1272 1273 switch (state) { 1274 case 0: 1275 if (cur_addr == old_addr + len) { 1276 /* direct RXOR */ 1277 order = 1; 1278 state = 1; 1279 } else if (old_addr == cur_addr + len) { 1280 /* reverse RXOR */ 1281 order = -1; 1282 state = 1; 1283 } else 1284 goto out; 1285 break; 1286 case 1: 1287 if ((i == src_cnt - 2) || 1288 (order == -1 && cur_addr != old_addr - len)) { 1289 order = 0; 1290 state = 0; 1291 } else if ((cur_addr == old_addr + len * order) || 1292 (cur_addr == old_addr + 2 * len) || 1293 (cur_addr == old_addr + 3 * len)) { 1294 state = 2; 1295 } else { 1296 order = 0; 1297 state = 0; 1298 } 1299 break; 1300 case 2: 1301 order = 0; 1302 state = 0; 1303 break; 1304 } 1305 } 1306 1307 out: 1308 if (state == 1 || state == 2) 1309 return 1; 1310 1311 return 0; 1312 } 1313 1314 /** 1315 * ppc440spe_adma_device_estimate - estimate the efficiency of processing 1316 * the operation given on this channel. It's assumed that 'chan' is 1317 * capable to process 'cap' type of operation. 1318 * @chan: channel to use 1319 * @cap: type of transaction 1320 * @dst_lst: array of destination pointers 1321 * @dst_cnt: number of destination operands 1322 * @src_lst: array of source pointers 1323 * @src_cnt: number of source operands 1324 * @src_sz: size of each source operand 1325 */ 1326 static int ppc440spe_adma_estimate(struct dma_chan *chan, 1327 enum dma_transaction_type cap, struct page **dst_lst, int dst_cnt, 1328 struct page **src_lst, int src_cnt, size_t src_sz) 1329 { 1330 int ef = 1; 1331 1332 if (cap == DMA_PQ || cap == DMA_PQ_VAL) { 1333 /* If RAID-6 capabilities were not activated don't try 1334 * to use them 1335 */ 1336 if (unlikely(!ppc440spe_r6_enabled)) 1337 return -1; 1338 } 1339 /* In the current implementation of ppc440spe ADMA driver it 1340 * makes sense to pick out only pq case, because it may be 1341 * processed: 1342 * (1) either using Biskup method on DMA2; 1343 * (2) or on DMA0/1. 1344 * Thus we give a favour to (1) if the sources are suitable; 1345 * else let it be processed on one of the DMA0/1 engines. 1346 * In the sum_product case where destination is also the 1347 * source process it on DMA0/1 only. 1348 */ 1349 if (cap == DMA_PQ && chan->chan_id == PPC440SPE_XOR_ID) { 1350 1351 if (dst_cnt == 1 && src_cnt == 2 && dst_lst[0] == src_lst[1]) 1352 ef = 0; /* sum_product case, process on DMA0/1 */ 1353 else if (ppc440spe_can_rxor(src_lst, src_cnt, src_sz)) 1354 ef = 3; /* override (DMA0/1 + idle) */ 1355 else 1356 ef = 0; /* can't process on DMA2 if !rxor */ 1357 } 1358 1359 /* channel idleness increases the priority */ 1360 if (likely(ef) && 1361 !ppc440spe_chan_is_busy(to_ppc440spe_adma_chan(chan))) 1362 ef++; 1363 1364 return ef; 1365 } 1366 1367 struct dma_chan * 1368 ppc440spe_async_tx_find_best_channel(enum dma_transaction_type cap, 1369 struct page **dst_lst, int dst_cnt, struct page **src_lst, 1370 int src_cnt, size_t src_sz) 1371 { 1372 struct dma_chan *best_chan = NULL; 1373 struct ppc_dma_chan_ref *ref; 1374 int best_rank = -1; 1375 1376 if (unlikely(!src_sz)) 1377 return NULL; 1378 if (src_sz > PAGE_SIZE) { 1379 /* 1380 * should a user of the api ever pass > PAGE_SIZE requests 1381 * we sort out cases where temporary page-sized buffers 1382 * are used. 1383 */ 1384 switch (cap) { 1385 case DMA_PQ: 1386 if (src_cnt == 1 && dst_lst[1] == src_lst[0]) 1387 return NULL; 1388 if (src_cnt == 2 && dst_lst[1] == src_lst[1]) 1389 return NULL; 1390 break; 1391 case DMA_PQ_VAL: 1392 case DMA_XOR_VAL: 1393 return NULL; 1394 default: 1395 break; 1396 } 1397 } 1398 1399 list_for_each_entry(ref, &ppc440spe_adma_chan_list, node) { 1400 if (dma_has_cap(cap, ref->chan->device->cap_mask)) { 1401 int rank; 1402 1403 rank = ppc440spe_adma_estimate(ref->chan, cap, dst_lst, 1404 dst_cnt, src_lst, src_cnt, src_sz); 1405 if (rank > best_rank) { 1406 best_rank = rank; 1407 best_chan = ref->chan; 1408 } 1409 } 1410 } 1411 1412 return best_chan; 1413 } 1414 EXPORT_SYMBOL_GPL(ppc440spe_async_tx_find_best_channel); 1415 1416 /** 1417 * ppc440spe_get_group_entry - get group entry with index idx 1418 * @tdesc: is the last allocated slot in the group. 1419 */ 1420 static struct ppc440spe_adma_desc_slot * 1421 ppc440spe_get_group_entry(struct ppc440spe_adma_desc_slot *tdesc, u32 entry_idx) 1422 { 1423 struct ppc440spe_adma_desc_slot *iter = tdesc->group_head; 1424 int i = 0; 1425 1426 if (entry_idx < 0 || entry_idx >= (tdesc->src_cnt + tdesc->dst_cnt)) { 1427 printk("%s: entry_idx %d, src_cnt %d, dst_cnt %d\n", 1428 __func__, entry_idx, tdesc->src_cnt, tdesc->dst_cnt); 1429 BUG(); 1430 } 1431 1432 list_for_each_entry(iter, &tdesc->group_list, chain_node) { 1433 if (i++ == entry_idx) 1434 break; 1435 } 1436 return iter; 1437 } 1438 1439 /** 1440 * ppc440spe_adma_free_slots - flags descriptor slots for reuse 1441 * @slot: Slot to free 1442 * Caller must hold &ppc440spe_chan->lock while calling this function 1443 */ 1444 static void ppc440spe_adma_free_slots(struct ppc440spe_adma_desc_slot *slot, 1445 struct ppc440spe_adma_chan *chan) 1446 { 1447 int stride = slot->slots_per_op; 1448 1449 while (stride--) { 1450 slot->slots_per_op = 0; 1451 slot = list_entry(slot->slot_node.next, 1452 struct ppc440spe_adma_desc_slot, 1453 slot_node); 1454 } 1455 } 1456 1457 /** 1458 * ppc440spe_adma_run_tx_complete_actions - call functions to be called 1459 * upon completion 1460 */ 1461 static dma_cookie_t ppc440spe_adma_run_tx_complete_actions( 1462 struct ppc440spe_adma_desc_slot *desc, 1463 struct ppc440spe_adma_chan *chan, 1464 dma_cookie_t cookie) 1465 { 1466 BUG_ON(desc->async_tx.cookie < 0); 1467 if (desc->async_tx.cookie > 0) { 1468 cookie = desc->async_tx.cookie; 1469 desc->async_tx.cookie = 0; 1470 1471 dma_descriptor_unmap(&desc->async_tx); 1472 /* call the callback (must not sleep or submit new 1473 * operations to this channel) 1474 */ 1475 dmaengine_desc_get_callback_invoke(&desc->async_tx, NULL); 1476 } 1477 1478 /* run dependent operations */ 1479 dma_run_dependencies(&desc->async_tx); 1480 1481 return cookie; 1482 } 1483 1484 /** 1485 * ppc440spe_adma_clean_slot - clean up CDB slot (if ack is set) 1486 */ 1487 static int ppc440spe_adma_clean_slot(struct ppc440spe_adma_desc_slot *desc, 1488 struct ppc440spe_adma_chan *chan) 1489 { 1490 /* the client is allowed to attach dependent operations 1491 * until 'ack' is set 1492 */ 1493 if (!async_tx_test_ack(&desc->async_tx)) 1494 return 0; 1495 1496 /* leave the last descriptor in the chain 1497 * so we can append to it 1498 */ 1499 if (list_is_last(&desc->chain_node, &chan->chain) || 1500 desc->phys == ppc440spe_chan_get_current_descriptor(chan)) 1501 return 1; 1502 1503 if (chan->device->id != PPC440SPE_XOR_ID) { 1504 /* our DMA interrupt handler clears opc field of 1505 * each processed descriptor. For all types of 1506 * operations except for ZeroSum we do not actually 1507 * need ack from the interrupt handler. ZeroSum is a 1508 * special case since the result of this operation 1509 * is available from the handler only, so if we see 1510 * such type of descriptor (which is unprocessed yet) 1511 * then leave it in chain. 1512 */ 1513 struct dma_cdb *cdb = desc->hw_desc; 1514 if (cdb->opc == DMA_CDB_OPC_DCHECK128) 1515 return 1; 1516 } 1517 1518 dev_dbg(chan->device->common.dev, "\tfree slot %llx: %d stride: %d\n", 1519 desc->phys, desc->idx, desc->slots_per_op); 1520 1521 list_del(&desc->chain_node); 1522 ppc440spe_adma_free_slots(desc, chan); 1523 return 0; 1524 } 1525 1526 /** 1527 * __ppc440spe_adma_slot_cleanup - this is the common clean-up routine 1528 * which runs through the channel CDBs list until reach the descriptor 1529 * currently processed. When routine determines that all CDBs of group 1530 * are completed then corresponding callbacks (if any) are called and slots 1531 * are freed. 1532 */ 1533 static void __ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1534 { 1535 struct ppc440spe_adma_desc_slot *iter, *_iter, *group_start = NULL; 1536 dma_cookie_t cookie = 0; 1537 u32 current_desc = ppc440spe_chan_get_current_descriptor(chan); 1538 int busy = ppc440spe_chan_is_busy(chan); 1539 int seen_current = 0, slot_cnt = 0, slots_per_op = 0; 1540 1541 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: %s\n", 1542 chan->device->id, __func__); 1543 1544 if (!current_desc) { 1545 /* There were no transactions yet, so 1546 * nothing to clean 1547 */ 1548 return; 1549 } 1550 1551 /* free completed slots from the chain starting with 1552 * the oldest descriptor 1553 */ 1554 list_for_each_entry_safe(iter, _iter, &chan->chain, 1555 chain_node) { 1556 dev_dbg(chan->device->common.dev, "\tcookie: %d slot: %d " 1557 "busy: %d this_desc: %#llx next_desc: %#x " 1558 "cur: %#x ack: %d\n", 1559 iter->async_tx.cookie, iter->idx, busy, iter->phys, 1560 ppc440spe_desc_get_link(iter, chan), current_desc, 1561 async_tx_test_ack(&iter->async_tx)); 1562 prefetch(_iter); 1563 prefetch(&_iter->async_tx); 1564 1565 /* do not advance past the current descriptor loaded into the 1566 * hardware channel,subsequent descriptors are either in process 1567 * or have not been submitted 1568 */ 1569 if (seen_current) 1570 break; 1571 1572 /* stop the search if we reach the current descriptor and the 1573 * channel is busy, or if it appears that the current descriptor 1574 * needs to be re-read (i.e. has been appended to) 1575 */ 1576 if (iter->phys == current_desc) { 1577 BUG_ON(seen_current++); 1578 if (busy || ppc440spe_desc_get_link(iter, chan)) { 1579 /* not all descriptors of the group have 1580 * been completed; exit. 1581 */ 1582 break; 1583 } 1584 } 1585 1586 /* detect the start of a group transaction */ 1587 if (!slot_cnt && !slots_per_op) { 1588 slot_cnt = iter->slot_cnt; 1589 slots_per_op = iter->slots_per_op; 1590 if (slot_cnt <= slots_per_op) { 1591 slot_cnt = 0; 1592 slots_per_op = 0; 1593 } 1594 } 1595 1596 if (slot_cnt) { 1597 if (!group_start) 1598 group_start = iter; 1599 slot_cnt -= slots_per_op; 1600 } 1601 1602 /* all the members of a group are complete */ 1603 if (slots_per_op != 0 && slot_cnt == 0) { 1604 struct ppc440spe_adma_desc_slot *grp_iter, *_grp_iter; 1605 int end_of_chain = 0; 1606 1607 /* clean up the group */ 1608 slot_cnt = group_start->slot_cnt; 1609 grp_iter = group_start; 1610 list_for_each_entry_safe_from(grp_iter, _grp_iter, 1611 &chan->chain, chain_node) { 1612 1613 cookie = ppc440spe_adma_run_tx_complete_actions( 1614 grp_iter, chan, cookie); 1615 1616 slot_cnt -= slots_per_op; 1617 end_of_chain = ppc440spe_adma_clean_slot( 1618 grp_iter, chan); 1619 if (end_of_chain && slot_cnt) { 1620 /* Should wait for ZeroSum completion */ 1621 if (cookie > 0) 1622 chan->common.completed_cookie = cookie; 1623 return; 1624 } 1625 1626 if (slot_cnt == 0 || end_of_chain) 1627 break; 1628 } 1629 1630 /* the group should be complete at this point */ 1631 BUG_ON(slot_cnt); 1632 1633 slots_per_op = 0; 1634 group_start = NULL; 1635 if (end_of_chain) 1636 break; 1637 else 1638 continue; 1639 } else if (slots_per_op) /* wait for group completion */ 1640 continue; 1641 1642 cookie = ppc440spe_adma_run_tx_complete_actions(iter, chan, 1643 cookie); 1644 1645 if (ppc440spe_adma_clean_slot(iter, chan)) 1646 break; 1647 } 1648 1649 BUG_ON(!seen_current); 1650 1651 if (cookie > 0) { 1652 chan->common.completed_cookie = cookie; 1653 pr_debug("\tcompleted cookie %d\n", cookie); 1654 } 1655 1656 } 1657 1658 /** 1659 * ppc440spe_adma_tasklet - clean up watch-dog initiator 1660 */ 1661 static void ppc440spe_adma_tasklet(struct tasklet_struct *t) 1662 { 1663 struct ppc440spe_adma_chan *chan = from_tasklet(chan, t, irq_tasklet); 1664 1665 spin_lock_nested(&chan->lock, SINGLE_DEPTH_NESTING); 1666 __ppc440spe_adma_slot_cleanup(chan); 1667 spin_unlock(&chan->lock); 1668 } 1669 1670 /** 1671 * ppc440spe_adma_slot_cleanup - clean up scheduled initiator 1672 */ 1673 static void ppc440spe_adma_slot_cleanup(struct ppc440spe_adma_chan *chan) 1674 { 1675 spin_lock_bh(&chan->lock); 1676 __ppc440spe_adma_slot_cleanup(chan); 1677 spin_unlock_bh(&chan->lock); 1678 } 1679 1680 /** 1681 * ppc440spe_adma_alloc_slots - allocate free slots (if any) 1682 */ 1683 static struct ppc440spe_adma_desc_slot *ppc440spe_adma_alloc_slots( 1684 struct ppc440spe_adma_chan *chan, int num_slots, 1685 int slots_per_op) 1686 { 1687 struct ppc440spe_adma_desc_slot *iter = NULL, *_iter; 1688 struct ppc440spe_adma_desc_slot *alloc_start = NULL; 1689 int slots_found, retry = 0; 1690 LIST_HEAD(chain); 1691 1692 1693 BUG_ON(!num_slots || !slots_per_op); 1694 /* start search from the last allocated descrtiptor 1695 * if a contiguous allocation can not be found start searching 1696 * from the beginning of the list 1697 */ 1698 retry: 1699 slots_found = 0; 1700 if (retry == 0) 1701 iter = chan->last_used; 1702 else 1703 iter = list_entry(&chan->all_slots, 1704 struct ppc440spe_adma_desc_slot, 1705 slot_node); 1706 list_for_each_entry_safe_continue(iter, _iter, &chan->all_slots, 1707 slot_node) { 1708 prefetch(_iter); 1709 prefetch(&_iter->async_tx); 1710 if (iter->slots_per_op) { 1711 slots_found = 0; 1712 continue; 1713 } 1714 1715 /* start the allocation if the slot is correctly aligned */ 1716 if (!slots_found++) 1717 alloc_start = iter; 1718 1719 if (slots_found == num_slots) { 1720 struct ppc440spe_adma_desc_slot *alloc_tail = NULL; 1721 struct ppc440spe_adma_desc_slot *last_used = NULL; 1722 1723 iter = alloc_start; 1724 while (num_slots) { 1725 int i; 1726 /* pre-ack all but the last descriptor */ 1727 if (num_slots != slots_per_op) 1728 async_tx_ack(&iter->async_tx); 1729 1730 list_add_tail(&iter->chain_node, &chain); 1731 alloc_tail = iter; 1732 iter->async_tx.cookie = 0; 1733 iter->hw_next = NULL; 1734 iter->flags = 0; 1735 iter->slot_cnt = num_slots; 1736 iter->xor_check_result = NULL; 1737 for (i = 0; i < slots_per_op; i++) { 1738 iter->slots_per_op = slots_per_op - i; 1739 last_used = iter; 1740 iter = list_entry(iter->slot_node.next, 1741 struct ppc440spe_adma_desc_slot, 1742 slot_node); 1743 } 1744 num_slots -= slots_per_op; 1745 } 1746 alloc_tail->group_head = alloc_start; 1747 alloc_tail->async_tx.cookie = -EBUSY; 1748 list_splice(&chain, &alloc_tail->group_list); 1749 chan->last_used = last_used; 1750 return alloc_tail; 1751 } 1752 } 1753 if (!retry++) 1754 goto retry; 1755 1756 /* try to free some slots if the allocation fails */ 1757 tasklet_schedule(&chan->irq_tasklet); 1758 return NULL; 1759 } 1760 1761 /** 1762 * ppc440spe_adma_alloc_chan_resources - allocate pools for CDB slots 1763 */ 1764 static int ppc440spe_adma_alloc_chan_resources(struct dma_chan *chan) 1765 { 1766 struct ppc440spe_adma_chan *ppc440spe_chan; 1767 struct ppc440spe_adma_desc_slot *slot = NULL; 1768 char *hw_desc; 1769 int i, db_sz; 1770 int init; 1771 1772 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1773 init = ppc440spe_chan->slots_allocated ? 0 : 1; 1774 chan->chan_id = ppc440spe_chan->device->id; 1775 1776 /* Allocate descriptor slots */ 1777 i = ppc440spe_chan->slots_allocated; 1778 if (ppc440spe_chan->device->id != PPC440SPE_XOR_ID) 1779 db_sz = sizeof(struct dma_cdb); 1780 else 1781 db_sz = sizeof(struct xor_cb); 1782 1783 for (; i < (ppc440spe_chan->device->pool_size / db_sz); i++) { 1784 slot = kzalloc_obj(struct ppc440spe_adma_desc_slot); 1785 if (!slot) { 1786 printk(KERN_INFO "SPE ADMA Channel only initialized" 1787 " %d descriptor slots", i--); 1788 break; 1789 } 1790 1791 hw_desc = (char *) ppc440spe_chan->device->dma_desc_pool_virt; 1792 slot->hw_desc = (void *) &hw_desc[i * db_sz]; 1793 dma_async_tx_descriptor_init(&slot->async_tx, chan); 1794 slot->async_tx.tx_submit = ppc440spe_adma_tx_submit; 1795 INIT_LIST_HEAD(&slot->chain_node); 1796 INIT_LIST_HEAD(&slot->slot_node); 1797 INIT_LIST_HEAD(&slot->group_list); 1798 slot->phys = ppc440spe_chan->device->dma_desc_pool + i * db_sz; 1799 slot->idx = i; 1800 1801 spin_lock_bh(&ppc440spe_chan->lock); 1802 ppc440spe_chan->slots_allocated++; 1803 list_add_tail(&slot->slot_node, &ppc440spe_chan->all_slots); 1804 spin_unlock_bh(&ppc440spe_chan->lock); 1805 } 1806 1807 if (i && !ppc440spe_chan->last_used) { 1808 ppc440spe_chan->last_used = 1809 list_entry(ppc440spe_chan->all_slots.next, 1810 struct ppc440spe_adma_desc_slot, 1811 slot_node); 1812 } 1813 1814 dev_dbg(ppc440spe_chan->device->common.dev, 1815 "ppc440spe adma%d: allocated %d descriptor slots\n", 1816 ppc440spe_chan->device->id, i); 1817 1818 /* initialize the channel and the chain with a null operation */ 1819 if (init) { 1820 switch (ppc440spe_chan->device->id) { 1821 case PPC440SPE_DMA0_ID: 1822 case PPC440SPE_DMA1_ID: 1823 ppc440spe_chan->hw_chain_inited = 0; 1824 /* Use WXOR for self-testing */ 1825 if (!ppc440spe_r6_tchan) 1826 ppc440spe_r6_tchan = ppc440spe_chan; 1827 break; 1828 case PPC440SPE_XOR_ID: 1829 ppc440spe_chan_start_null_xor(ppc440spe_chan); 1830 break; 1831 default: 1832 BUG(); 1833 } 1834 ppc440spe_chan->needs_unmap = 1; 1835 } 1836 1837 return (i > 0) ? i : -ENOMEM; 1838 } 1839 1840 /** 1841 * ppc440spe_rxor_set_region_data - 1842 */ 1843 static void ppc440spe_rxor_set_region(struct ppc440spe_adma_desc_slot *desc, 1844 u8 xor_arg_no, u32 mask) 1845 { 1846 struct xor_cb *xcb = desc->hw_desc; 1847 1848 xcb->ops[xor_arg_no].h |= mask; 1849 } 1850 1851 /** 1852 * ppc440spe_rxor_set_src - 1853 */ 1854 static void ppc440spe_rxor_set_src(struct ppc440spe_adma_desc_slot *desc, 1855 u8 xor_arg_no, dma_addr_t addr) 1856 { 1857 struct xor_cb *xcb = desc->hw_desc; 1858 1859 xcb->ops[xor_arg_no].h |= DMA_CUED_XOR_BASE; 1860 xcb->ops[xor_arg_no].l = addr; 1861 } 1862 1863 /** 1864 * ppc440spe_rxor_set_mult - 1865 */ 1866 static void ppc440spe_rxor_set_mult(struct ppc440spe_adma_desc_slot *desc, 1867 u8 xor_arg_no, u8 idx, u8 mult) 1868 { 1869 struct xor_cb *xcb = desc->hw_desc; 1870 1871 xcb->ops[xor_arg_no].h |= mult << (DMA_CUED_MULT1_OFF + idx * 8); 1872 } 1873 1874 /** 1875 * ppc440spe_adma_check_threshold - append CDBs to h/w chain if threshold 1876 * has been achieved 1877 */ 1878 static void ppc440spe_adma_check_threshold(struct ppc440spe_adma_chan *chan) 1879 { 1880 dev_dbg(chan->device->common.dev, "ppc440spe adma%d: pending: %d\n", 1881 chan->device->id, chan->pending); 1882 1883 if (chan->pending >= PPC440SPE_ADMA_THRESHOLD) { 1884 chan->pending = 0; 1885 ppc440spe_chan_append(chan); 1886 } 1887 } 1888 1889 /** 1890 * ppc440spe_adma_tx_submit - submit new descriptor group to the channel 1891 * (it's not necessary that descriptors will be submitted to the h/w 1892 * chains too right now) 1893 */ 1894 static dma_cookie_t ppc440spe_adma_tx_submit(struct dma_async_tx_descriptor *tx) 1895 { 1896 struct ppc440spe_adma_desc_slot *sw_desc; 1897 struct ppc440spe_adma_chan *chan = to_ppc440spe_adma_chan(tx->chan); 1898 struct ppc440spe_adma_desc_slot *group_start, *old_chain_tail; 1899 int slot_cnt; 1900 int slots_per_op; 1901 dma_cookie_t cookie; 1902 1903 sw_desc = tx_to_ppc440spe_adma_slot(tx); 1904 1905 group_start = sw_desc->group_head; 1906 slot_cnt = group_start->slot_cnt; 1907 slots_per_op = group_start->slots_per_op; 1908 1909 spin_lock_bh(&chan->lock); 1910 cookie = dma_cookie_assign(tx); 1911 1912 if (unlikely(list_empty(&chan->chain))) { 1913 /* first peer */ 1914 list_splice_init(&sw_desc->group_list, &chan->chain); 1915 chan_first_cdb[chan->device->id] = group_start; 1916 } else { 1917 /* isn't first peer, bind CDBs to chain */ 1918 old_chain_tail = list_entry(chan->chain.prev, 1919 struct ppc440spe_adma_desc_slot, 1920 chain_node); 1921 list_splice_init(&sw_desc->group_list, 1922 &old_chain_tail->chain_node); 1923 /* fix up the hardware chain */ 1924 ppc440spe_desc_set_link(chan, old_chain_tail, group_start); 1925 } 1926 1927 /* increment the pending count by the number of operations */ 1928 chan->pending += slot_cnt / slots_per_op; 1929 ppc440spe_adma_check_threshold(chan); 1930 spin_unlock_bh(&chan->lock); 1931 1932 dev_dbg(chan->device->common.dev, 1933 "ppc440spe adma%d: %s cookie: %d slot: %d tx %p\n", 1934 chan->device->id, __func__, 1935 sw_desc->async_tx.cookie, sw_desc->idx, sw_desc); 1936 1937 return cookie; 1938 } 1939 1940 /** 1941 * ppc440spe_adma_prep_dma_interrupt - prepare CDB for a pseudo DMA operation 1942 */ 1943 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_interrupt( 1944 struct dma_chan *chan, unsigned long flags) 1945 { 1946 struct ppc440spe_adma_chan *ppc440spe_chan; 1947 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1948 int slot_cnt, slots_per_op; 1949 1950 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1951 1952 dev_dbg(ppc440spe_chan->device->common.dev, 1953 "ppc440spe adma%d: %s\n", ppc440spe_chan->device->id, 1954 __func__); 1955 1956 spin_lock_bh(&ppc440spe_chan->lock); 1957 slot_cnt = slots_per_op = 1; 1958 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1959 slots_per_op); 1960 if (sw_desc) { 1961 group_start = sw_desc->group_head; 1962 ppc440spe_desc_init_interrupt(group_start, ppc440spe_chan); 1963 group_start->unmap_len = 0; 1964 sw_desc->async_tx.flags = flags; 1965 } 1966 spin_unlock_bh(&ppc440spe_chan->lock); 1967 1968 return sw_desc ? &sw_desc->async_tx : NULL; 1969 } 1970 1971 /** 1972 * ppc440spe_adma_prep_dma_memcpy - prepare CDB for a MEMCPY operation 1973 */ 1974 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_memcpy( 1975 struct dma_chan *chan, dma_addr_t dma_dest, 1976 dma_addr_t dma_src, size_t len, unsigned long flags) 1977 { 1978 struct ppc440spe_adma_chan *ppc440spe_chan; 1979 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 1980 int slot_cnt, slots_per_op; 1981 1982 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 1983 1984 if (unlikely(!len)) 1985 return NULL; 1986 1987 BUG_ON(len > PPC440SPE_ADMA_DMA_MAX_BYTE_COUNT); 1988 1989 spin_lock_bh(&ppc440spe_chan->lock); 1990 1991 dev_dbg(ppc440spe_chan->device->common.dev, 1992 "ppc440spe adma%d: %s len: %u int_en %d\n", 1993 ppc440spe_chan->device->id, __func__, len, 1994 flags & DMA_PREP_INTERRUPT ? 1 : 0); 1995 slot_cnt = slots_per_op = 1; 1996 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1997 slots_per_op); 1998 if (sw_desc) { 1999 group_start = sw_desc->group_head; 2000 ppc440spe_desc_init_memcpy(group_start, flags); 2001 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2002 ppc440spe_adma_memcpy_xor_set_src(group_start, dma_src, 0); 2003 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2004 sw_desc->unmap_len = len; 2005 sw_desc->async_tx.flags = flags; 2006 } 2007 spin_unlock_bh(&ppc440spe_chan->lock); 2008 2009 return sw_desc ? &sw_desc->async_tx : NULL; 2010 } 2011 2012 /** 2013 * ppc440spe_adma_prep_dma_xor - prepare CDB for a XOR operation 2014 */ 2015 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor( 2016 struct dma_chan *chan, dma_addr_t dma_dest, 2017 dma_addr_t *dma_src, u32 src_cnt, size_t len, 2018 unsigned long flags) 2019 { 2020 struct ppc440spe_adma_chan *ppc440spe_chan; 2021 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 2022 int slot_cnt, slots_per_op; 2023 2024 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2025 2026 ADMA_LL_DBG(prep_dma_xor_dbg(ppc440spe_chan->device->id, 2027 dma_dest, dma_src, src_cnt)); 2028 if (unlikely(!len)) 2029 return NULL; 2030 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2031 2032 dev_dbg(ppc440spe_chan->device->common.dev, 2033 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2034 ppc440spe_chan->device->id, __func__, src_cnt, len, 2035 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2036 2037 spin_lock_bh(&ppc440spe_chan->lock); 2038 slot_cnt = ppc440spe_chan_xor_slot_count(len, src_cnt, &slots_per_op); 2039 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2040 slots_per_op); 2041 if (sw_desc) { 2042 group_start = sw_desc->group_head; 2043 ppc440spe_desc_init_xor(group_start, src_cnt, flags); 2044 ppc440spe_adma_set_dest(group_start, dma_dest, 0); 2045 while (src_cnt--) 2046 ppc440spe_adma_memcpy_xor_set_src(group_start, 2047 dma_src[src_cnt], src_cnt); 2048 ppc440spe_desc_set_byte_count(group_start, ppc440spe_chan, len); 2049 sw_desc->unmap_len = len; 2050 sw_desc->async_tx.flags = flags; 2051 } 2052 spin_unlock_bh(&ppc440spe_chan->lock); 2053 2054 return sw_desc ? &sw_desc->async_tx : NULL; 2055 } 2056 2057 static inline void 2058 ppc440spe_desc_set_xor_src_cnt(struct ppc440spe_adma_desc_slot *desc, 2059 int src_cnt); 2060 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor); 2061 2062 /** 2063 * ppc440spe_adma_init_dma2rxor_slot - 2064 */ 2065 static void ppc440spe_adma_init_dma2rxor_slot( 2066 struct ppc440spe_adma_desc_slot *desc, 2067 dma_addr_t *src, int src_cnt) 2068 { 2069 int i; 2070 2071 /* initialize CDB */ 2072 for (i = 0; i < src_cnt; i++) { 2073 ppc440spe_adma_dma2rxor_prep_src(desc, &desc->rxor_cursor, i, 2074 desc->src_cnt, (u32)src[i]); 2075 } 2076 } 2077 2078 /** 2079 * ppc440spe_dma01_prep_mult - 2080 * for Q operation where destination is also the source 2081 */ 2082 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_mult( 2083 struct ppc440spe_adma_chan *ppc440spe_chan, 2084 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2085 const unsigned char *scf, size_t len, unsigned long flags) 2086 { 2087 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2088 unsigned long op = 0; 2089 int slot_cnt; 2090 2091 set_bit(PPC440SPE_DESC_WXOR, &op); 2092 slot_cnt = 2; 2093 2094 spin_lock_bh(&ppc440spe_chan->lock); 2095 2096 /* use WXOR, each descriptor occupies one slot */ 2097 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2098 if (sw_desc) { 2099 struct ppc440spe_adma_chan *chan; 2100 struct ppc440spe_adma_desc_slot *iter; 2101 struct dma_cdb *hw_desc; 2102 2103 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2104 set_bits(op, &sw_desc->flags); 2105 sw_desc->src_cnt = src_cnt; 2106 sw_desc->dst_cnt = dst_cnt; 2107 /* First descriptor, zero data in the destination and copy it 2108 * to q page using MULTICAST transfer. 2109 */ 2110 iter = list_first_entry(&sw_desc->group_list, 2111 struct ppc440spe_adma_desc_slot, 2112 chain_node); 2113 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2114 /* set 'next' pointer */ 2115 iter->hw_next = list_entry(iter->chain_node.next, 2116 struct ppc440spe_adma_desc_slot, 2117 chain_node); 2118 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2119 hw_desc = iter->hw_desc; 2120 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2121 2122 ppc440spe_desc_set_dest_addr(iter, chan, 2123 DMA_CUED_XOR_BASE, dst[0], 0); 2124 ppc440spe_desc_set_dest_addr(iter, chan, 0, dst[1], 1); 2125 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2126 src[0]); 2127 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2128 iter->unmap_len = len; 2129 2130 /* 2131 * Second descriptor, multiply data from the q page 2132 * and store the result in real destination. 2133 */ 2134 iter = list_first_entry(&iter->chain_node, 2135 struct ppc440spe_adma_desc_slot, 2136 chain_node); 2137 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2138 iter->hw_next = NULL; 2139 if (flags & DMA_PREP_INTERRUPT) 2140 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2141 else 2142 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2143 2144 hw_desc = iter->hw_desc; 2145 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2146 ppc440spe_desc_set_src_addr(iter, chan, 0, 2147 DMA_CUED_XOR_HB, dst[1]); 2148 ppc440spe_desc_set_dest_addr(iter, chan, 2149 DMA_CUED_XOR_BASE, dst[0], 0); 2150 2151 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2152 DMA_CDB_SG_DST1, scf[0]); 2153 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2154 iter->unmap_len = len; 2155 sw_desc->async_tx.flags = flags; 2156 } 2157 2158 spin_unlock_bh(&ppc440spe_chan->lock); 2159 2160 return sw_desc; 2161 } 2162 2163 /** 2164 * ppc440spe_dma01_prep_sum_product - 2165 * Dx = A*(P+Pxy) + B*(Q+Qxy) operation where destination is also 2166 * the source. 2167 */ 2168 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_sum_product( 2169 struct ppc440spe_adma_chan *ppc440spe_chan, 2170 dma_addr_t *dst, dma_addr_t *src, int src_cnt, 2171 const unsigned char *scf, size_t len, unsigned long flags) 2172 { 2173 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2174 unsigned long op = 0; 2175 int slot_cnt; 2176 2177 set_bit(PPC440SPE_DESC_WXOR, &op); 2178 slot_cnt = 3; 2179 2180 spin_lock_bh(&ppc440spe_chan->lock); 2181 2182 /* WXOR, each descriptor occupies one slot */ 2183 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2184 if (sw_desc) { 2185 struct ppc440spe_adma_chan *chan; 2186 struct ppc440spe_adma_desc_slot *iter; 2187 struct dma_cdb *hw_desc; 2188 2189 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2190 set_bits(op, &sw_desc->flags); 2191 sw_desc->src_cnt = src_cnt; 2192 sw_desc->dst_cnt = 1; 2193 /* 1st descriptor, src[1] data to q page and zero destination */ 2194 iter = list_first_entry(&sw_desc->group_list, 2195 struct ppc440spe_adma_desc_slot, 2196 chain_node); 2197 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2198 iter->hw_next = list_entry(iter->chain_node.next, 2199 struct ppc440spe_adma_desc_slot, 2200 chain_node); 2201 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2202 hw_desc = iter->hw_desc; 2203 hw_desc->opc = DMA_CDB_OPC_MULTICAST; 2204 2205 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2206 *dst, 0); 2207 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2208 ppc440spe_chan->qdest, 1); 2209 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2210 src[1]); 2211 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2212 iter->unmap_len = len; 2213 2214 /* 2nd descriptor, multiply src[1] data and store the 2215 * result in destination */ 2216 iter = list_first_entry(&iter->chain_node, 2217 struct ppc440spe_adma_desc_slot, 2218 chain_node); 2219 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2220 /* set 'next' pointer */ 2221 iter->hw_next = list_entry(iter->chain_node.next, 2222 struct ppc440spe_adma_desc_slot, 2223 chain_node); 2224 if (flags & DMA_PREP_INTERRUPT) 2225 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2226 else 2227 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2228 2229 hw_desc = iter->hw_desc; 2230 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2231 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2232 ppc440spe_chan->qdest); 2233 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2234 *dst, 0); 2235 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2236 DMA_CDB_SG_DST1, scf[1]); 2237 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2238 iter->unmap_len = len; 2239 2240 /* 2241 * 3rd descriptor, multiply src[0] data and xor it 2242 * with destination 2243 */ 2244 iter = list_first_entry(&iter->chain_node, 2245 struct ppc440spe_adma_desc_slot, 2246 chain_node); 2247 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2248 iter->hw_next = NULL; 2249 if (flags & DMA_PREP_INTERRUPT) 2250 set_bit(PPC440SPE_DESC_INT, &iter->flags); 2251 else 2252 clear_bit(PPC440SPE_DESC_INT, &iter->flags); 2253 2254 hw_desc = iter->hw_desc; 2255 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2256 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, 2257 src[0]); 2258 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, 2259 *dst, 0); 2260 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2261 DMA_CDB_SG_DST1, scf[0]); 2262 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, len); 2263 iter->unmap_len = len; 2264 sw_desc->async_tx.flags = flags; 2265 } 2266 2267 spin_unlock_bh(&ppc440spe_chan->lock); 2268 2269 return sw_desc; 2270 } 2271 2272 static struct ppc440spe_adma_desc_slot *ppc440spe_dma01_prep_pq( 2273 struct ppc440spe_adma_chan *ppc440spe_chan, 2274 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2275 const unsigned char *scf, size_t len, unsigned long flags) 2276 { 2277 int slot_cnt; 2278 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2279 unsigned long op = 0; 2280 unsigned char mult = 1; 2281 2282 pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2283 __func__, dst_cnt, src_cnt, len); 2284 /* select operations WXOR/RXOR depending on the 2285 * source addresses of operators and the number 2286 * of destinations (RXOR support only Q-parity calculations) 2287 */ 2288 set_bit(PPC440SPE_DESC_WXOR, &op); 2289 if (!test_and_set_bit(PPC440SPE_RXOR_RUN, &ppc440spe_rxor_state)) { 2290 /* no active RXOR; 2291 * do RXOR if: 2292 * - there are more than 1 source, 2293 * - len is aligned on 512-byte boundary, 2294 * - source addresses fit to one of 4 possible regions. 2295 */ 2296 if (src_cnt > 1 && 2297 !(len & MQ0_CF2H_RXOR_BS_MASK) && 2298 (src[0] + len) == src[1]) { 2299 /* may do RXOR R1 R2 */ 2300 set_bit(PPC440SPE_DESC_RXOR, &op); 2301 if (src_cnt != 2) { 2302 /* may try to enhance region of RXOR */ 2303 if ((src[1] + len) == src[2]) { 2304 /* do RXOR R1 R2 R3 */ 2305 set_bit(PPC440SPE_DESC_RXOR123, 2306 &op); 2307 } else if ((src[1] + len * 2) == src[2]) { 2308 /* do RXOR R1 R2 R4 */ 2309 set_bit(PPC440SPE_DESC_RXOR124, &op); 2310 } else if ((src[1] + len * 3) == src[2]) { 2311 /* do RXOR R1 R2 R5 */ 2312 set_bit(PPC440SPE_DESC_RXOR125, 2313 &op); 2314 } else { 2315 /* do RXOR R1 R2 */ 2316 set_bit(PPC440SPE_DESC_RXOR12, 2317 &op); 2318 } 2319 } else { 2320 /* do RXOR R1 R2 */ 2321 set_bit(PPC440SPE_DESC_RXOR12, &op); 2322 } 2323 } 2324 2325 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2326 /* can not do this operation with RXOR */ 2327 clear_bit(PPC440SPE_RXOR_RUN, 2328 &ppc440spe_rxor_state); 2329 } else { 2330 /* can do; set block size right now */ 2331 ppc440spe_desc_set_rxor_block_size(len); 2332 } 2333 } 2334 2335 /* Number of necessary slots depends on operation type selected */ 2336 if (!test_bit(PPC440SPE_DESC_RXOR, &op)) { 2337 /* This is a WXOR only chain. Need descriptors for each 2338 * source to GF-XOR them with WXOR, and need descriptors 2339 * for each destination to zero them with WXOR 2340 */ 2341 slot_cnt = src_cnt; 2342 2343 if (flags & DMA_PREP_ZERO_P) { 2344 slot_cnt++; 2345 set_bit(PPC440SPE_ZERO_P, &op); 2346 } 2347 if (flags & DMA_PREP_ZERO_Q) { 2348 slot_cnt++; 2349 set_bit(PPC440SPE_ZERO_Q, &op); 2350 } 2351 } else { 2352 /* Need 1/2 descriptor for RXOR operation, and 2353 * need (src_cnt - (2 or 3)) for WXOR of sources 2354 * remained (if any) 2355 */ 2356 slot_cnt = dst_cnt; 2357 2358 if (flags & DMA_PREP_ZERO_P) 2359 set_bit(PPC440SPE_ZERO_P, &op); 2360 if (flags & DMA_PREP_ZERO_Q) 2361 set_bit(PPC440SPE_ZERO_Q, &op); 2362 2363 if (test_bit(PPC440SPE_DESC_RXOR12, &op)) 2364 slot_cnt += src_cnt - 2; 2365 else 2366 slot_cnt += src_cnt - 3; 2367 2368 /* Thus we have either RXOR only chain or 2369 * mixed RXOR/WXOR 2370 */ 2371 if (slot_cnt == dst_cnt) 2372 /* RXOR only chain */ 2373 clear_bit(PPC440SPE_DESC_WXOR, &op); 2374 } 2375 2376 spin_lock_bh(&ppc440spe_chan->lock); 2377 /* for both RXOR/WXOR each descriptor occupies one slot */ 2378 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2379 if (sw_desc) { 2380 ppc440spe_desc_init_dma01pq(sw_desc, dst_cnt, src_cnt, 2381 flags, op); 2382 2383 /* setup dst/src/mult */ 2384 pr_debug("%s: set dst descriptor 0, 1: 0x%016llx, 0x%016llx\n", 2385 __func__, dst[0], dst[1]); 2386 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2387 while (src_cnt--) { 2388 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2389 src_cnt); 2390 2391 /* NOTE: "Multi = 0 is equivalent to = 1" as it 2392 * stated in 440SPSPe_RAID6_Addendum_UM_1_17.pdf 2393 * doesn't work for RXOR with DMA0/1! Instead, multi=0 2394 * leads to zeroing source data after RXOR. 2395 * So, for P case set-up mult=1 explicitly. 2396 */ 2397 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2398 mult = scf[src_cnt]; 2399 ppc440spe_adma_pq_set_src_mult(sw_desc, 2400 mult, src_cnt, dst_cnt - 1); 2401 } 2402 2403 /* Setup byte count foreach slot just allocated */ 2404 sw_desc->async_tx.flags = flags; 2405 list_for_each_entry(iter, &sw_desc->group_list, 2406 chain_node) { 2407 ppc440spe_desc_set_byte_count(iter, 2408 ppc440spe_chan, len); 2409 iter->unmap_len = len; 2410 } 2411 } 2412 spin_unlock_bh(&ppc440spe_chan->lock); 2413 2414 return sw_desc; 2415 } 2416 2417 static struct ppc440spe_adma_desc_slot *ppc440spe_dma2_prep_pq( 2418 struct ppc440spe_adma_chan *ppc440spe_chan, 2419 dma_addr_t *dst, int dst_cnt, dma_addr_t *src, int src_cnt, 2420 const unsigned char *scf, size_t len, unsigned long flags) 2421 { 2422 int slot_cnt, descs_per_op; 2423 struct ppc440spe_adma_desc_slot *sw_desc = NULL, *iter; 2424 unsigned long op = 0; 2425 unsigned char mult = 1; 2426 2427 BUG_ON(!dst_cnt); 2428 /*pr_debug("%s: dst_cnt %d, src_cnt %d, len %d\n", 2429 __func__, dst_cnt, src_cnt, len);*/ 2430 2431 spin_lock_bh(&ppc440spe_chan->lock); 2432 descs_per_op = ppc440spe_dma2_pq_slot_count(src, src_cnt, len); 2433 if (descs_per_op < 0) { 2434 spin_unlock_bh(&ppc440spe_chan->lock); 2435 return NULL; 2436 } 2437 2438 /* depending on number of sources we have 1 or 2 RXOR chains */ 2439 slot_cnt = descs_per_op * dst_cnt; 2440 2441 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 1); 2442 if (sw_desc) { 2443 op = slot_cnt; 2444 sw_desc->async_tx.flags = flags; 2445 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2446 ppc440spe_desc_init_dma2pq(iter, dst_cnt, src_cnt, 2447 --op ? 0 : flags); 2448 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2449 len); 2450 iter->unmap_len = len; 2451 2452 ppc440spe_init_rxor_cursor(&(iter->rxor_cursor)); 2453 iter->rxor_cursor.len = len; 2454 iter->descs_per_op = descs_per_op; 2455 } 2456 op = 0; 2457 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2458 op++; 2459 if (op % descs_per_op == 0) 2460 ppc440spe_adma_init_dma2rxor_slot(iter, src, 2461 src_cnt); 2462 if (likely(!list_is_last(&iter->chain_node, 2463 &sw_desc->group_list))) { 2464 /* set 'next' pointer */ 2465 iter->hw_next = 2466 list_entry(iter->chain_node.next, 2467 struct ppc440spe_adma_desc_slot, 2468 chain_node); 2469 ppc440spe_xor_set_link(iter, iter->hw_next); 2470 } else { 2471 /* this is the last descriptor. */ 2472 iter->hw_next = NULL; 2473 } 2474 } 2475 2476 /* fixup head descriptor */ 2477 sw_desc->dst_cnt = dst_cnt; 2478 if (flags & DMA_PREP_ZERO_P) 2479 set_bit(PPC440SPE_ZERO_P, &sw_desc->flags); 2480 if (flags & DMA_PREP_ZERO_Q) 2481 set_bit(PPC440SPE_ZERO_Q, &sw_desc->flags); 2482 2483 /* setup dst/src/mult */ 2484 ppc440spe_adma_pq_set_dest(sw_desc, dst, flags); 2485 2486 while (src_cnt--) { 2487 /* handle descriptors (if dst_cnt == 2) inside 2488 * the ppc440spe_adma_pq_set_srcxxx() functions 2489 */ 2490 ppc440spe_adma_pq_set_src(sw_desc, src[src_cnt], 2491 src_cnt); 2492 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) 2493 mult = scf[src_cnt]; 2494 ppc440spe_adma_pq_set_src_mult(sw_desc, 2495 mult, src_cnt, dst_cnt - 1); 2496 } 2497 } 2498 spin_unlock_bh(&ppc440spe_chan->lock); 2499 ppc440spe_desc_set_rxor_block_size(len); 2500 return sw_desc; 2501 } 2502 2503 /** 2504 * ppc440spe_adma_prep_dma_pq - prepare CDB (group) for a GF-XOR operation 2505 */ 2506 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pq( 2507 struct dma_chan *chan, dma_addr_t *dst, dma_addr_t *src, 2508 unsigned int src_cnt, const unsigned char *scf, 2509 size_t len, unsigned long flags) 2510 { 2511 struct ppc440spe_adma_chan *ppc440spe_chan; 2512 struct ppc440spe_adma_desc_slot *sw_desc = NULL; 2513 int dst_cnt = 0; 2514 2515 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2516 2517 ADMA_LL_DBG(prep_dma_pq_dbg(ppc440spe_chan->device->id, 2518 dst, src, src_cnt)); 2519 BUG_ON(!len); 2520 BUG_ON(len > PPC440SPE_ADMA_XOR_MAX_BYTE_COUNT); 2521 BUG_ON(!src_cnt); 2522 2523 if (src_cnt == 1 && dst[1] == src[0]) { 2524 dma_addr_t dest[2]; 2525 2526 /* dst[1] is real destination (Q) */ 2527 dest[0] = dst[1]; 2528 /* this is the page to multicast source data to */ 2529 dest[1] = ppc440spe_chan->qdest; 2530 sw_desc = ppc440spe_dma01_prep_mult(ppc440spe_chan, 2531 dest, 2, src, src_cnt, scf, len, flags); 2532 return sw_desc ? &sw_desc->async_tx : NULL; 2533 } 2534 2535 if (src_cnt == 2 && dst[1] == src[1]) { 2536 sw_desc = ppc440spe_dma01_prep_sum_product(ppc440spe_chan, 2537 &dst[1], src, 2, scf, len, flags); 2538 return sw_desc ? &sw_desc->async_tx : NULL; 2539 } 2540 2541 if (!(flags & DMA_PREP_PQ_DISABLE_P)) { 2542 BUG_ON(!dst[0]); 2543 dst_cnt++; 2544 flags |= DMA_PREP_ZERO_P; 2545 } 2546 2547 if (!(flags & DMA_PREP_PQ_DISABLE_Q)) { 2548 BUG_ON(!dst[1]); 2549 dst_cnt++; 2550 flags |= DMA_PREP_ZERO_Q; 2551 } 2552 2553 BUG_ON(!dst_cnt); 2554 2555 dev_dbg(ppc440spe_chan->device->common.dev, 2556 "ppc440spe adma%d: %s src_cnt: %d len: %u int_en: %d\n", 2557 ppc440spe_chan->device->id, __func__, src_cnt, len, 2558 flags & DMA_PREP_INTERRUPT ? 1 : 0); 2559 2560 switch (ppc440spe_chan->device->id) { 2561 case PPC440SPE_DMA0_ID: 2562 case PPC440SPE_DMA1_ID: 2563 sw_desc = ppc440spe_dma01_prep_pq(ppc440spe_chan, 2564 dst, dst_cnt, src, src_cnt, scf, 2565 len, flags); 2566 break; 2567 2568 case PPC440SPE_XOR_ID: 2569 sw_desc = ppc440spe_dma2_prep_pq(ppc440spe_chan, 2570 dst, dst_cnt, src, src_cnt, scf, 2571 len, flags); 2572 break; 2573 } 2574 2575 return sw_desc ? &sw_desc->async_tx : NULL; 2576 } 2577 2578 /** 2579 * ppc440spe_adma_prep_dma_pqzero_sum - prepare CDB group for 2580 * a PQ_ZERO_SUM operation 2581 */ 2582 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_pqzero_sum( 2583 struct dma_chan *chan, dma_addr_t *pq, dma_addr_t *src, 2584 unsigned int src_cnt, const unsigned char *scf, size_t len, 2585 enum sum_check_flags *pqres, unsigned long flags) 2586 { 2587 struct ppc440spe_adma_chan *ppc440spe_chan; 2588 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 2589 dma_addr_t pdest, qdest; 2590 int slot_cnt, slots_per_op, idst, dst_cnt; 2591 2592 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 2593 2594 if (flags & DMA_PREP_PQ_DISABLE_P) 2595 pdest = 0; 2596 else 2597 pdest = pq[0]; 2598 2599 if (flags & DMA_PREP_PQ_DISABLE_Q) 2600 qdest = 0; 2601 else 2602 qdest = pq[1]; 2603 2604 ADMA_LL_DBG(prep_dma_pqzero_sum_dbg(ppc440spe_chan->device->id, 2605 src, src_cnt, scf)); 2606 2607 /* Always use WXOR for P/Q calculations (two destinations). 2608 * Need 1 or 2 extra slots to verify results are zero. 2609 */ 2610 idst = dst_cnt = (pdest && qdest) ? 2 : 1; 2611 2612 /* One additional slot per destination to clone P/Q 2613 * before calculation (we have to preserve destinations). 2614 */ 2615 slot_cnt = src_cnt + dst_cnt * 2; 2616 slots_per_op = 1; 2617 2618 spin_lock_bh(&ppc440spe_chan->lock); 2619 sw_desc = ppc440spe_adma_alloc_slots(ppc440spe_chan, slot_cnt, 2620 slots_per_op); 2621 if (sw_desc) { 2622 ppc440spe_desc_init_dma01pqzero_sum(sw_desc, dst_cnt, src_cnt); 2623 2624 /* Setup byte count for each slot just allocated */ 2625 sw_desc->async_tx.flags = flags; 2626 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 2627 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2628 len); 2629 iter->unmap_len = len; 2630 } 2631 2632 if (pdest) { 2633 struct dma_cdb *hw_desc; 2634 struct ppc440spe_adma_chan *chan; 2635 2636 iter = sw_desc->group_head; 2637 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2638 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2639 iter->hw_next = list_entry(iter->chain_node.next, 2640 struct ppc440spe_adma_desc_slot, 2641 chain_node); 2642 hw_desc = iter->hw_desc; 2643 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2644 iter->src_cnt = 0; 2645 iter->dst_cnt = 0; 2646 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2647 ppc440spe_chan->pdest, 0); 2648 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, pdest); 2649 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2650 len); 2651 iter->unmap_len = 0; 2652 /* override pdest to preserve original P */ 2653 pdest = ppc440spe_chan->pdest; 2654 } 2655 if (qdest) { 2656 struct dma_cdb *hw_desc; 2657 struct ppc440spe_adma_chan *chan; 2658 2659 iter = list_first_entry(&sw_desc->group_list, 2660 struct ppc440spe_adma_desc_slot, 2661 chain_node); 2662 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2663 2664 if (pdest) { 2665 iter = list_entry(iter->chain_node.next, 2666 struct ppc440spe_adma_desc_slot, 2667 chain_node); 2668 } 2669 2670 memset(iter->hw_desc, 0, sizeof(struct dma_cdb)); 2671 iter->hw_next = list_entry(iter->chain_node.next, 2672 struct ppc440spe_adma_desc_slot, 2673 chain_node); 2674 hw_desc = iter->hw_desc; 2675 hw_desc->opc = DMA_CDB_OPC_MV_SG1_SG2; 2676 iter->src_cnt = 0; 2677 iter->dst_cnt = 0; 2678 ppc440spe_desc_set_dest_addr(iter, chan, 0, 2679 ppc440spe_chan->qdest, 0); 2680 ppc440spe_desc_set_src_addr(iter, chan, 0, 0, qdest); 2681 ppc440spe_desc_set_byte_count(iter, ppc440spe_chan, 2682 len); 2683 iter->unmap_len = 0; 2684 /* override qdest to preserve original Q */ 2685 qdest = ppc440spe_chan->qdest; 2686 } 2687 2688 /* Setup destinations for P/Q ops */ 2689 ppc440spe_adma_pqzero_sum_set_dest(sw_desc, pdest, qdest); 2690 2691 /* Setup zero QWORDs into DCHECK CDBs */ 2692 idst = dst_cnt; 2693 list_for_each_entry_reverse(iter, &sw_desc->group_list, 2694 chain_node) { 2695 /* 2696 * The last CDB corresponds to Q-parity check, 2697 * the one before last CDB corresponds 2698 * P-parity check 2699 */ 2700 if (idst == DMA_DEST_MAX_NUM) { 2701 if (idst == dst_cnt) { 2702 set_bit(PPC440SPE_DESC_QCHECK, 2703 &iter->flags); 2704 } else { 2705 set_bit(PPC440SPE_DESC_PCHECK, 2706 &iter->flags); 2707 } 2708 } else { 2709 if (qdest) { 2710 set_bit(PPC440SPE_DESC_QCHECK, 2711 &iter->flags); 2712 } else { 2713 set_bit(PPC440SPE_DESC_PCHECK, 2714 &iter->flags); 2715 } 2716 } 2717 iter->xor_check_result = pqres; 2718 2719 /* 2720 * set it to zero, if check fail then result will 2721 * be updated 2722 */ 2723 *iter->xor_check_result = 0; 2724 ppc440spe_desc_set_dcheck(iter, ppc440spe_chan, 2725 ppc440spe_qword); 2726 2727 if (!(--dst_cnt)) 2728 break; 2729 } 2730 2731 /* Setup sources and mults for P/Q ops */ 2732 list_for_each_entry_continue_reverse(iter, &sw_desc->group_list, 2733 chain_node) { 2734 struct ppc440spe_adma_chan *chan; 2735 u32 mult_dst; 2736 2737 chan = to_ppc440spe_adma_chan(iter->async_tx.chan); 2738 ppc440spe_desc_set_src_addr(iter, chan, 0, 2739 DMA_CUED_XOR_HB, 2740 src[src_cnt - 1]); 2741 if (qdest) { 2742 mult_dst = (dst_cnt - 1) ? DMA_CDB_SG_DST2 : 2743 DMA_CDB_SG_DST1; 2744 ppc440spe_desc_set_src_mult(iter, chan, 2745 DMA_CUED_MULT1_OFF, 2746 mult_dst, 2747 scf[src_cnt - 1]); 2748 } 2749 if (!(--src_cnt)) 2750 break; 2751 } 2752 } 2753 spin_unlock_bh(&ppc440spe_chan->lock); 2754 return sw_desc ? &sw_desc->async_tx : NULL; 2755 } 2756 2757 /** 2758 * ppc440spe_adma_prep_dma_xor_zero_sum - prepare CDB group for 2759 * XOR ZERO_SUM operation 2760 */ 2761 static struct dma_async_tx_descriptor *ppc440spe_adma_prep_dma_xor_zero_sum( 2762 struct dma_chan *chan, dma_addr_t *src, unsigned int src_cnt, 2763 size_t len, enum sum_check_flags *result, unsigned long flags) 2764 { 2765 struct dma_async_tx_descriptor *tx; 2766 dma_addr_t pq[2]; 2767 2768 /* validate P, disable Q */ 2769 pq[0] = src[0]; 2770 pq[1] = 0; 2771 flags |= DMA_PREP_PQ_DISABLE_Q; 2772 2773 tx = ppc440spe_adma_prep_dma_pqzero_sum(chan, pq, &src[1], 2774 src_cnt - 1, 0, len, 2775 result, flags); 2776 return tx; 2777 } 2778 2779 /** 2780 * ppc440spe_adma_set_dest - set destination address into descriptor 2781 */ 2782 static void ppc440spe_adma_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2783 dma_addr_t addr, int index) 2784 { 2785 struct ppc440spe_adma_chan *chan; 2786 2787 BUG_ON(index >= sw_desc->dst_cnt); 2788 2789 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2790 2791 switch (chan->device->id) { 2792 case PPC440SPE_DMA0_ID: 2793 case PPC440SPE_DMA1_ID: 2794 /* to do: support transfers lengths > 2795 * PPC440SPE_ADMA_DMA/XOR_MAX_BYTE_COUNT 2796 */ 2797 ppc440spe_desc_set_dest_addr(sw_desc->group_head, 2798 chan, 0, addr, index); 2799 break; 2800 case PPC440SPE_XOR_ID: 2801 sw_desc = ppc440spe_get_group_entry(sw_desc, index); 2802 ppc440spe_desc_set_dest_addr(sw_desc, 2803 chan, 0, addr, index); 2804 break; 2805 } 2806 } 2807 2808 static void ppc440spe_adma_pq_zero_op(struct ppc440spe_adma_desc_slot *iter, 2809 struct ppc440spe_adma_chan *chan, dma_addr_t addr) 2810 { 2811 /* To clear destinations update the descriptor 2812 * (P or Q depending on index) as follows: 2813 * addr is destination (0 corresponds to SG2): 2814 */ 2815 ppc440spe_desc_set_dest_addr(iter, chan, DMA_CUED_XOR_BASE, addr, 0); 2816 2817 /* ... and the addr is source: */ 2818 ppc440spe_desc_set_src_addr(iter, chan, 0, DMA_CUED_XOR_HB, addr); 2819 2820 /* addr is always SG2 then the mult is always DST1 */ 2821 ppc440spe_desc_set_src_mult(iter, chan, DMA_CUED_MULT1_OFF, 2822 DMA_CDB_SG_DST1, 1); 2823 } 2824 2825 /** 2826 * ppc440spe_adma_pq_set_dest - set destination address into descriptor 2827 * for the PQXOR operation 2828 */ 2829 static void ppc440spe_adma_pq_set_dest(struct ppc440spe_adma_desc_slot *sw_desc, 2830 dma_addr_t *addrs, unsigned long flags) 2831 { 2832 struct ppc440spe_adma_desc_slot *iter; 2833 struct ppc440spe_adma_chan *chan; 2834 dma_addr_t paddr, qaddr; 2835 dma_addr_t addr = 0, ppath, qpath; 2836 int index = 0, i; 2837 2838 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 2839 2840 if (flags & DMA_PREP_PQ_DISABLE_P) 2841 paddr = 0; 2842 else 2843 paddr = addrs[0]; 2844 2845 if (flags & DMA_PREP_PQ_DISABLE_Q) 2846 qaddr = 0; 2847 else 2848 qaddr = addrs[1]; 2849 2850 if (!paddr || !qaddr) 2851 addr = paddr ? paddr : qaddr; 2852 2853 switch (chan->device->id) { 2854 case PPC440SPE_DMA0_ID: 2855 case PPC440SPE_DMA1_ID: 2856 /* walk through the WXOR source list and set P/Q-destinations 2857 * for each slot: 2858 */ 2859 if (!test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 2860 /* This is WXOR-only chain; may have 1/2 zero descs */ 2861 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 2862 index++; 2863 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 2864 index++; 2865 2866 iter = ppc440spe_get_group_entry(sw_desc, index); 2867 if (addr) { 2868 /* one destination */ 2869 list_for_each_entry_from(iter, 2870 &sw_desc->group_list, chain_node) 2871 ppc440spe_desc_set_dest_addr(iter, chan, 2872 DMA_CUED_XOR_BASE, addr, 0); 2873 } else { 2874 /* two destinations */ 2875 list_for_each_entry_from(iter, 2876 &sw_desc->group_list, chain_node) { 2877 ppc440spe_desc_set_dest_addr(iter, chan, 2878 DMA_CUED_XOR_BASE, paddr, 0); 2879 ppc440spe_desc_set_dest_addr(iter, chan, 2880 DMA_CUED_XOR_BASE, qaddr, 1); 2881 } 2882 } 2883 2884 if (index) { 2885 /* To clear destinations update the descriptor 2886 * (1st,2nd, or both depending on flags) 2887 */ 2888 index = 0; 2889 if (test_bit(PPC440SPE_ZERO_P, 2890 &sw_desc->flags)) { 2891 iter = ppc440spe_get_group_entry( 2892 sw_desc, index++); 2893 ppc440spe_adma_pq_zero_op(iter, chan, 2894 paddr); 2895 } 2896 2897 if (test_bit(PPC440SPE_ZERO_Q, 2898 &sw_desc->flags)) { 2899 iter = ppc440spe_get_group_entry( 2900 sw_desc, index++); 2901 ppc440spe_adma_pq_zero_op(iter, chan, 2902 qaddr); 2903 } 2904 2905 return; 2906 } 2907 } else { 2908 /* This is RXOR-only or RXOR/WXOR mixed chain */ 2909 2910 /* If we want to include destination into calculations, 2911 * then make dest addresses cued with mult=1 (XOR). 2912 */ 2913 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2914 DMA_CUED_XOR_HB : 2915 DMA_CUED_XOR_BASE | 2916 (1 << DMA_CUED_MULT1_OFF); 2917 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2918 DMA_CUED_XOR_HB : 2919 DMA_CUED_XOR_BASE | 2920 (1 << DMA_CUED_MULT1_OFF); 2921 2922 /* Setup destination(s) in RXOR slot(s) */ 2923 iter = ppc440spe_get_group_entry(sw_desc, index++); 2924 ppc440spe_desc_set_dest_addr(iter, chan, 2925 paddr ? ppath : qpath, 2926 paddr ? paddr : qaddr, 0); 2927 if (!addr) { 2928 /* two destinations */ 2929 iter = ppc440spe_get_group_entry(sw_desc, 2930 index++); 2931 ppc440spe_desc_set_dest_addr(iter, chan, 2932 qpath, qaddr, 0); 2933 } 2934 2935 if (test_bit(PPC440SPE_DESC_WXOR, &sw_desc->flags)) { 2936 /* Setup destination(s) in remaining WXOR 2937 * slots 2938 */ 2939 iter = ppc440spe_get_group_entry(sw_desc, 2940 index); 2941 if (addr) { 2942 /* one destination */ 2943 list_for_each_entry_from(iter, 2944 &sw_desc->group_list, 2945 chain_node) 2946 ppc440spe_desc_set_dest_addr( 2947 iter, chan, 2948 DMA_CUED_XOR_BASE, 2949 addr, 0); 2950 2951 } else { 2952 /* two destinations */ 2953 list_for_each_entry_from(iter, 2954 &sw_desc->group_list, 2955 chain_node) { 2956 ppc440spe_desc_set_dest_addr( 2957 iter, chan, 2958 DMA_CUED_XOR_BASE, 2959 paddr, 0); 2960 ppc440spe_desc_set_dest_addr( 2961 iter, chan, 2962 DMA_CUED_XOR_BASE, 2963 qaddr, 1); 2964 } 2965 } 2966 } 2967 2968 } 2969 break; 2970 2971 case PPC440SPE_XOR_ID: 2972 /* DMA2 descriptors have only 1 destination, so there are 2973 * two chains - one for each dest. 2974 * If we want to include destination into calculations, 2975 * then make dest addresses cued with mult=1 (XOR). 2976 */ 2977 ppath = test_bit(PPC440SPE_ZERO_P, &sw_desc->flags) ? 2978 DMA_CUED_XOR_HB : 2979 DMA_CUED_XOR_BASE | 2980 (1 << DMA_CUED_MULT1_OFF); 2981 2982 qpath = test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags) ? 2983 DMA_CUED_XOR_HB : 2984 DMA_CUED_XOR_BASE | 2985 (1 << DMA_CUED_MULT1_OFF); 2986 2987 iter = ppc440spe_get_group_entry(sw_desc, 0); 2988 for (i = 0; i < sw_desc->descs_per_op; i++) { 2989 ppc440spe_desc_set_dest_addr(iter, chan, 2990 paddr ? ppath : qpath, 2991 paddr ? paddr : qaddr, 0); 2992 iter = list_entry(iter->chain_node.next, 2993 struct ppc440spe_adma_desc_slot, 2994 chain_node); 2995 } 2996 2997 if (!addr) { 2998 /* Two destinations; setup Q here */ 2999 iter = ppc440spe_get_group_entry(sw_desc, 3000 sw_desc->descs_per_op); 3001 for (i = 0; i < sw_desc->descs_per_op; i++) { 3002 ppc440spe_desc_set_dest_addr(iter, 3003 chan, qpath, qaddr, 0); 3004 iter = list_entry(iter->chain_node.next, 3005 struct ppc440spe_adma_desc_slot, 3006 chain_node); 3007 } 3008 } 3009 3010 break; 3011 } 3012 } 3013 3014 /** 3015 * ppc440spe_adma_pq_zero_sum_set_dest - set destination address into descriptor 3016 * for the PQ_ZERO_SUM operation 3017 */ 3018 static void ppc440spe_adma_pqzero_sum_set_dest( 3019 struct ppc440spe_adma_desc_slot *sw_desc, 3020 dma_addr_t paddr, dma_addr_t qaddr) 3021 { 3022 struct ppc440spe_adma_desc_slot *iter, *end; 3023 struct ppc440spe_adma_chan *chan; 3024 dma_addr_t addr = 0; 3025 int idx; 3026 3027 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3028 3029 /* walk through the WXOR source list and set P/Q-destinations 3030 * for each slot 3031 */ 3032 idx = (paddr && qaddr) ? 2 : 1; 3033 /* set end */ 3034 list_for_each_entry_reverse(end, &sw_desc->group_list, 3035 chain_node) { 3036 if (!(--idx)) 3037 break; 3038 } 3039 /* set start */ 3040 idx = (paddr && qaddr) ? 2 : 1; 3041 iter = ppc440spe_get_group_entry(sw_desc, idx); 3042 3043 if (paddr && qaddr) { 3044 /* two destinations */ 3045 list_for_each_entry_from(iter, &sw_desc->group_list, 3046 chain_node) { 3047 if (unlikely(iter == end)) 3048 break; 3049 ppc440spe_desc_set_dest_addr(iter, chan, 3050 DMA_CUED_XOR_BASE, paddr, 0); 3051 ppc440spe_desc_set_dest_addr(iter, chan, 3052 DMA_CUED_XOR_BASE, qaddr, 1); 3053 } 3054 } else { 3055 /* one destination */ 3056 addr = paddr ? paddr : qaddr; 3057 list_for_each_entry_from(iter, &sw_desc->group_list, 3058 chain_node) { 3059 if (unlikely(iter == end)) 3060 break; 3061 ppc440spe_desc_set_dest_addr(iter, chan, 3062 DMA_CUED_XOR_BASE, addr, 0); 3063 } 3064 } 3065 3066 /* The remaining descriptors are DATACHECK. These have no need in 3067 * destination. Actually, these destinations are used there 3068 * as sources for check operation. So, set addr as source. 3069 */ 3070 ppc440spe_desc_set_src_addr(end, chan, 0, 0, addr ? addr : paddr); 3071 3072 if (!addr) { 3073 end = list_entry(end->chain_node.next, 3074 struct ppc440spe_adma_desc_slot, chain_node); 3075 ppc440spe_desc_set_src_addr(end, chan, 0, 0, qaddr); 3076 } 3077 } 3078 3079 /** 3080 * ppc440spe_desc_set_xor_src_cnt - set source count into descriptor 3081 */ 3082 static inline void ppc440spe_desc_set_xor_src_cnt( 3083 struct ppc440spe_adma_desc_slot *desc, 3084 int src_cnt) 3085 { 3086 struct xor_cb *hw_desc = desc->hw_desc; 3087 3088 hw_desc->cbc &= ~XOR_CDCR_OAC_MSK; 3089 hw_desc->cbc |= src_cnt; 3090 } 3091 3092 /** 3093 * ppc440spe_adma_pq_set_src - set source address into descriptor 3094 */ 3095 static void ppc440spe_adma_pq_set_src(struct ppc440spe_adma_desc_slot *sw_desc, 3096 dma_addr_t addr, int index) 3097 { 3098 struct ppc440spe_adma_chan *chan; 3099 dma_addr_t haddr = 0; 3100 struct ppc440spe_adma_desc_slot *iter = NULL; 3101 3102 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3103 3104 switch (chan->device->id) { 3105 case PPC440SPE_DMA0_ID: 3106 case PPC440SPE_DMA1_ID: 3107 /* DMA0,1 may do: WXOR, RXOR, RXOR+WXORs chain 3108 */ 3109 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3110 /* RXOR-only or RXOR/WXOR operation */ 3111 int iskip = test_bit(PPC440SPE_DESC_RXOR12, 3112 &sw_desc->flags) ? 2 : 3; 3113 3114 if (index == 0) { 3115 /* 1st slot (RXOR) */ 3116 /* setup sources region (R1-2-3, R1-2-4, 3117 * or R1-2-5) 3118 */ 3119 if (test_bit(PPC440SPE_DESC_RXOR12, 3120 &sw_desc->flags)) 3121 haddr = DMA_RXOR12 << 3122 DMA_CUED_REGION_OFF; 3123 else if (test_bit(PPC440SPE_DESC_RXOR123, 3124 &sw_desc->flags)) 3125 haddr = DMA_RXOR123 << 3126 DMA_CUED_REGION_OFF; 3127 else if (test_bit(PPC440SPE_DESC_RXOR124, 3128 &sw_desc->flags)) 3129 haddr = DMA_RXOR124 << 3130 DMA_CUED_REGION_OFF; 3131 else if (test_bit(PPC440SPE_DESC_RXOR125, 3132 &sw_desc->flags)) 3133 haddr = DMA_RXOR125 << 3134 DMA_CUED_REGION_OFF; 3135 else 3136 BUG(); 3137 haddr |= DMA_CUED_XOR_BASE; 3138 iter = ppc440spe_get_group_entry(sw_desc, 0); 3139 } else if (index < iskip) { 3140 /* 1st slot (RXOR) 3141 * shall actually set source address only once 3142 * instead of first <iskip> 3143 */ 3144 iter = NULL; 3145 } else { 3146 /* 2nd/3d and next slots (WXOR); 3147 * skip first slot with RXOR 3148 */ 3149 haddr = DMA_CUED_XOR_HB; 3150 iter = ppc440spe_get_group_entry(sw_desc, 3151 index - iskip + sw_desc->dst_cnt); 3152 } 3153 } else { 3154 int znum = 0; 3155 3156 /* WXOR-only operation; skip first slots with 3157 * zeroing destinations 3158 */ 3159 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3160 znum++; 3161 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3162 znum++; 3163 3164 haddr = DMA_CUED_XOR_HB; 3165 iter = ppc440spe_get_group_entry(sw_desc, 3166 index + znum); 3167 } 3168 3169 if (likely(iter)) { 3170 ppc440spe_desc_set_src_addr(iter, chan, 0, haddr, addr); 3171 3172 if (!index && 3173 test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags) && 3174 sw_desc->dst_cnt == 2) { 3175 /* if we have two destinations for RXOR, then 3176 * setup source in the second descr too 3177 */ 3178 iter = ppc440spe_get_group_entry(sw_desc, 1); 3179 ppc440spe_desc_set_src_addr(iter, chan, 0, 3180 haddr, addr); 3181 } 3182 } 3183 break; 3184 3185 case PPC440SPE_XOR_ID: 3186 /* DMA2 may do Biskup */ 3187 iter = sw_desc->group_head; 3188 if (iter->dst_cnt == 2) { 3189 /* both P & Q calculations required; set P src here */ 3190 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3191 3192 /* this is for Q */ 3193 iter = ppc440spe_get_group_entry(sw_desc, 3194 sw_desc->descs_per_op); 3195 } 3196 ppc440spe_adma_dma2rxor_set_src(iter, index, addr); 3197 break; 3198 } 3199 } 3200 3201 /** 3202 * ppc440spe_adma_memcpy_xor_set_src - set source address into descriptor 3203 */ 3204 static void ppc440spe_adma_memcpy_xor_set_src( 3205 struct ppc440spe_adma_desc_slot *sw_desc, 3206 dma_addr_t addr, int index) 3207 { 3208 struct ppc440spe_adma_chan *chan; 3209 3210 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3211 sw_desc = sw_desc->group_head; 3212 3213 if (likely(sw_desc)) 3214 ppc440spe_desc_set_src_addr(sw_desc, chan, index, 0, addr); 3215 } 3216 3217 /** 3218 * ppc440spe_adma_dma2rxor_inc_addr - 3219 */ 3220 static void ppc440spe_adma_dma2rxor_inc_addr( 3221 struct ppc440spe_adma_desc_slot *desc, 3222 struct ppc440spe_rxor *cursor, int index, int src_cnt) 3223 { 3224 cursor->addr_count++; 3225 if (index == src_cnt - 1) { 3226 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3227 } else if (cursor->addr_count == XOR_MAX_OPS) { 3228 ppc440spe_desc_set_xor_src_cnt(desc, cursor->addr_count); 3229 cursor->addr_count = 0; 3230 cursor->desc_count++; 3231 } 3232 } 3233 3234 /** 3235 * ppc440spe_adma_dma2rxor_prep_src - setup RXOR types in DMA2 CDB 3236 */ 3237 static int ppc440spe_adma_dma2rxor_prep_src( 3238 struct ppc440spe_adma_desc_slot *hdesc, 3239 struct ppc440spe_rxor *cursor, int index, 3240 int src_cnt, u32 addr) 3241 { 3242 u32 sign; 3243 struct ppc440spe_adma_desc_slot *desc = hdesc; 3244 int i; 3245 3246 for (i = 0; i < cursor->desc_count; i++) { 3247 desc = list_entry(hdesc->chain_node.next, 3248 struct ppc440spe_adma_desc_slot, 3249 chain_node); 3250 } 3251 3252 switch (cursor->state) { 3253 case 0: 3254 if (addr == cursor->addrl + cursor->len) { 3255 /* direct RXOR */ 3256 cursor->state = 1; 3257 cursor->xor_count++; 3258 if (index == src_cnt-1) { 3259 ppc440spe_rxor_set_region(desc, 3260 cursor->addr_count, 3261 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3262 ppc440spe_adma_dma2rxor_inc_addr( 3263 desc, cursor, index, src_cnt); 3264 } 3265 } else if (cursor->addrl == addr + cursor->len) { 3266 /* reverse RXOR */ 3267 cursor->state = 1; 3268 cursor->xor_count++; 3269 set_bit(cursor->addr_count, &desc->reverse_flags[0]); 3270 if (index == src_cnt-1) { 3271 ppc440spe_rxor_set_region(desc, 3272 cursor->addr_count, 3273 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3274 ppc440spe_adma_dma2rxor_inc_addr( 3275 desc, cursor, index, src_cnt); 3276 } 3277 } else { 3278 printk(KERN_ERR "Cannot build " 3279 "DMA2 RXOR command block.\n"); 3280 BUG(); 3281 } 3282 break; 3283 case 1: 3284 sign = test_bit(cursor->addr_count, 3285 desc->reverse_flags) 3286 ? -1 : 1; 3287 if (index == src_cnt-2 || (sign == -1 3288 && addr != cursor->addrl - 2*cursor->len)) { 3289 cursor->state = 0; 3290 cursor->xor_count = 1; 3291 cursor->addrl = addr; 3292 ppc440spe_rxor_set_region(desc, 3293 cursor->addr_count, 3294 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3295 ppc440spe_adma_dma2rxor_inc_addr( 3296 desc, cursor, index, src_cnt); 3297 } else if (addr == cursor->addrl + 2*sign*cursor->len) { 3298 cursor->state = 2; 3299 cursor->xor_count = 0; 3300 ppc440spe_rxor_set_region(desc, 3301 cursor->addr_count, 3302 DMA_RXOR123 << DMA_CUED_REGION_OFF); 3303 if (index == src_cnt-1) { 3304 ppc440spe_adma_dma2rxor_inc_addr( 3305 desc, cursor, index, src_cnt); 3306 } 3307 } else if (addr == cursor->addrl + 3*cursor->len) { 3308 cursor->state = 2; 3309 cursor->xor_count = 0; 3310 ppc440spe_rxor_set_region(desc, 3311 cursor->addr_count, 3312 DMA_RXOR124 << DMA_CUED_REGION_OFF); 3313 if (index == src_cnt-1) { 3314 ppc440spe_adma_dma2rxor_inc_addr( 3315 desc, cursor, index, src_cnt); 3316 } 3317 } else if (addr == cursor->addrl + 4*cursor->len) { 3318 cursor->state = 2; 3319 cursor->xor_count = 0; 3320 ppc440spe_rxor_set_region(desc, 3321 cursor->addr_count, 3322 DMA_RXOR125 << DMA_CUED_REGION_OFF); 3323 if (index == src_cnt-1) { 3324 ppc440spe_adma_dma2rxor_inc_addr( 3325 desc, cursor, index, src_cnt); 3326 } 3327 } else { 3328 cursor->state = 0; 3329 cursor->xor_count = 1; 3330 cursor->addrl = addr; 3331 ppc440spe_rxor_set_region(desc, 3332 cursor->addr_count, 3333 DMA_RXOR12 << DMA_CUED_REGION_OFF); 3334 ppc440spe_adma_dma2rxor_inc_addr( 3335 desc, cursor, index, src_cnt); 3336 } 3337 break; 3338 case 2: 3339 cursor->state = 0; 3340 cursor->addrl = addr; 3341 cursor->xor_count++; 3342 if (index) { 3343 ppc440spe_adma_dma2rxor_inc_addr( 3344 desc, cursor, index, src_cnt); 3345 } 3346 break; 3347 } 3348 3349 return 0; 3350 } 3351 3352 /** 3353 * ppc440spe_adma_dma2rxor_set_src - set RXOR source address; it's assumed that 3354 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3355 */ 3356 static void ppc440spe_adma_dma2rxor_set_src( 3357 struct ppc440spe_adma_desc_slot *desc, 3358 int index, dma_addr_t addr) 3359 { 3360 struct xor_cb *xcb = desc->hw_desc; 3361 int k = 0, op = 0, lop = 0; 3362 3363 /* get the RXOR operand which corresponds to index addr */ 3364 while (op <= index) { 3365 lop = op; 3366 if (k == XOR_MAX_OPS) { 3367 k = 0; 3368 desc = list_entry(desc->chain_node.next, 3369 struct ppc440spe_adma_desc_slot, chain_node); 3370 xcb = desc->hw_desc; 3371 3372 } 3373 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3374 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3375 op += 2; 3376 else 3377 op += 3; 3378 } 3379 3380 BUG_ON(k < 1); 3381 3382 if (test_bit(k-1, desc->reverse_flags)) { 3383 /* reverse operand order; put last op in RXOR group */ 3384 if (index == op - 1) 3385 ppc440spe_rxor_set_src(desc, k - 1, addr); 3386 } else { 3387 /* direct operand order; put first op in RXOR group */ 3388 if (index == lop) 3389 ppc440spe_rxor_set_src(desc, k - 1, addr); 3390 } 3391 } 3392 3393 /** 3394 * ppc440spe_adma_dma2rxor_set_mult - set RXOR multipliers; it's assumed that 3395 * ppc440spe_adma_dma2rxor_prep_src() has already done prior this call 3396 */ 3397 static void ppc440spe_adma_dma2rxor_set_mult( 3398 struct ppc440spe_adma_desc_slot *desc, 3399 int index, u8 mult) 3400 { 3401 struct xor_cb *xcb = desc->hw_desc; 3402 int k = 0, op = 0, lop = 0; 3403 3404 /* get the RXOR operand which corresponds to index mult */ 3405 while (op <= index) { 3406 lop = op; 3407 if (k == XOR_MAX_OPS) { 3408 k = 0; 3409 desc = list_entry(desc->chain_node.next, 3410 struct ppc440spe_adma_desc_slot, 3411 chain_node); 3412 xcb = desc->hw_desc; 3413 3414 } 3415 if ((xcb->ops[k++].h & (DMA_RXOR12 << DMA_CUED_REGION_OFF)) == 3416 (DMA_RXOR12 << DMA_CUED_REGION_OFF)) 3417 op += 2; 3418 else 3419 op += 3; 3420 } 3421 3422 BUG_ON(k < 1); 3423 if (test_bit(k-1, desc->reverse_flags)) { 3424 /* reverse order */ 3425 ppc440spe_rxor_set_mult(desc, k - 1, op - index - 1, mult); 3426 } else { 3427 /* direct order */ 3428 ppc440spe_rxor_set_mult(desc, k - 1, index - lop, mult); 3429 } 3430 } 3431 3432 /** 3433 * ppc440spe_init_rxor_cursor - 3434 */ 3435 static void ppc440spe_init_rxor_cursor(struct ppc440spe_rxor *cursor) 3436 { 3437 memset(cursor, 0, sizeof(struct ppc440spe_rxor)); 3438 cursor->state = 2; 3439 } 3440 3441 /** 3442 * ppc440spe_adma_pq_set_src_mult - set multiplication coefficient into 3443 * descriptor for the PQXOR operation 3444 */ 3445 static void ppc440spe_adma_pq_set_src_mult( 3446 struct ppc440spe_adma_desc_slot *sw_desc, 3447 unsigned char mult, int index, int dst_pos) 3448 { 3449 struct ppc440spe_adma_chan *chan; 3450 u32 mult_idx, mult_dst; 3451 struct ppc440spe_adma_desc_slot *iter = NULL, *iter1 = NULL; 3452 3453 chan = to_ppc440spe_adma_chan(sw_desc->async_tx.chan); 3454 3455 switch (chan->device->id) { 3456 case PPC440SPE_DMA0_ID: 3457 case PPC440SPE_DMA1_ID: 3458 if (test_bit(PPC440SPE_DESC_RXOR, &sw_desc->flags)) { 3459 int region = test_bit(PPC440SPE_DESC_RXOR12, 3460 &sw_desc->flags) ? 2 : 3; 3461 3462 if (index < region) { 3463 /* RXOR multipliers */ 3464 iter = ppc440spe_get_group_entry(sw_desc, 3465 sw_desc->dst_cnt - 1); 3466 if (sw_desc->dst_cnt == 2) 3467 iter1 = ppc440spe_get_group_entry( 3468 sw_desc, 0); 3469 3470 mult_idx = DMA_CUED_MULT1_OFF + (index << 3); 3471 mult_dst = DMA_CDB_SG_SRC; 3472 } else { 3473 /* WXOR multiplier */ 3474 iter = ppc440spe_get_group_entry(sw_desc, 3475 index - region + 3476 sw_desc->dst_cnt); 3477 mult_idx = DMA_CUED_MULT1_OFF; 3478 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : 3479 DMA_CDB_SG_DST1; 3480 } 3481 } else { 3482 int znum = 0; 3483 3484 /* WXOR-only; 3485 * skip first slots with destinations (if ZERO_DST has 3486 * place) 3487 */ 3488 if (test_bit(PPC440SPE_ZERO_P, &sw_desc->flags)) 3489 znum++; 3490 if (test_bit(PPC440SPE_ZERO_Q, &sw_desc->flags)) 3491 znum++; 3492 3493 iter = ppc440spe_get_group_entry(sw_desc, index + znum); 3494 mult_idx = DMA_CUED_MULT1_OFF; 3495 mult_dst = dst_pos ? DMA_CDB_SG_DST2 : DMA_CDB_SG_DST1; 3496 } 3497 3498 if (likely(iter)) { 3499 ppc440spe_desc_set_src_mult(iter, chan, 3500 mult_idx, mult_dst, mult); 3501 3502 if (unlikely(iter1)) { 3503 /* if we have two destinations for RXOR, then 3504 * we've just set Q mult. Set-up P now. 3505 */ 3506 ppc440spe_desc_set_src_mult(iter1, chan, 3507 mult_idx, mult_dst, 1); 3508 } 3509 3510 } 3511 break; 3512 3513 case PPC440SPE_XOR_ID: 3514 iter = sw_desc->group_head; 3515 if (sw_desc->dst_cnt == 2) { 3516 /* both P & Q calculations required; set P mult here */ 3517 ppc440spe_adma_dma2rxor_set_mult(iter, index, 1); 3518 3519 /* and then set Q mult */ 3520 iter = ppc440spe_get_group_entry(sw_desc, 3521 sw_desc->descs_per_op); 3522 } 3523 ppc440spe_adma_dma2rxor_set_mult(iter, index, mult); 3524 break; 3525 } 3526 } 3527 3528 /** 3529 * ppc440spe_adma_free_chan_resources - free the resources allocated 3530 */ 3531 static void ppc440spe_adma_free_chan_resources(struct dma_chan *chan) 3532 { 3533 struct ppc440spe_adma_chan *ppc440spe_chan; 3534 struct ppc440spe_adma_desc_slot *iter, *_iter; 3535 int in_use_descs = 0; 3536 3537 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3538 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3539 3540 spin_lock_bh(&ppc440spe_chan->lock); 3541 list_for_each_entry_safe(iter, _iter, &ppc440spe_chan->chain, 3542 chain_node) { 3543 in_use_descs++; 3544 list_del(&iter->chain_node); 3545 } 3546 list_for_each_entry_safe_reverse(iter, _iter, 3547 &ppc440spe_chan->all_slots, slot_node) { 3548 list_del(&iter->slot_node); 3549 kfree(iter); 3550 ppc440spe_chan->slots_allocated--; 3551 } 3552 ppc440spe_chan->last_used = NULL; 3553 3554 dev_dbg(ppc440spe_chan->device->common.dev, 3555 "ppc440spe adma%d %s slots_allocated %d\n", 3556 ppc440spe_chan->device->id, 3557 __func__, ppc440spe_chan->slots_allocated); 3558 spin_unlock_bh(&ppc440spe_chan->lock); 3559 3560 /* one is ok since we left it on there on purpose */ 3561 if (in_use_descs > 1) 3562 printk(KERN_ERR "SPE: Freeing %d in use descriptors!\n", 3563 in_use_descs - 1); 3564 } 3565 3566 /** 3567 * ppc440spe_adma_tx_status - poll the status of an ADMA transaction 3568 * @chan: ADMA channel handle 3569 * @cookie: ADMA transaction identifier 3570 * @txstate: a holder for the current state of the channel 3571 */ 3572 static enum dma_status ppc440spe_adma_tx_status(struct dma_chan *chan, 3573 dma_cookie_t cookie, struct dma_tx_state *txstate) 3574 { 3575 struct ppc440spe_adma_chan *ppc440spe_chan; 3576 enum dma_status ret; 3577 3578 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3579 ret = dma_cookie_status(chan, cookie, txstate); 3580 if (ret == DMA_COMPLETE) 3581 return ret; 3582 3583 ppc440spe_adma_slot_cleanup(ppc440spe_chan); 3584 3585 return dma_cookie_status(chan, cookie, txstate); 3586 } 3587 3588 /** 3589 * ppc440spe_adma_eot_handler - end of transfer interrupt handler 3590 */ 3591 static irqreturn_t ppc440spe_adma_eot_handler(int irq, void *data) 3592 { 3593 struct ppc440spe_adma_chan *chan = data; 3594 3595 dev_dbg(chan->device->common.dev, 3596 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3597 3598 tasklet_schedule(&chan->irq_tasklet); 3599 ppc440spe_adma_device_clear_eot_status(chan); 3600 3601 return IRQ_HANDLED; 3602 } 3603 3604 /** 3605 * ppc440spe_adma_err_handler - DMA error interrupt handler; 3606 * do the same things as a eot handler 3607 */ 3608 static irqreturn_t ppc440spe_adma_err_handler(int irq, void *data) 3609 { 3610 struct ppc440spe_adma_chan *chan = data; 3611 3612 dev_dbg(chan->device->common.dev, 3613 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3614 3615 tasklet_schedule(&chan->irq_tasklet); 3616 ppc440spe_adma_device_clear_eot_status(chan); 3617 3618 return IRQ_HANDLED; 3619 } 3620 3621 /** 3622 * ppc440spe_test_callback - called when test operation has been done 3623 */ 3624 static void ppc440spe_test_callback(void *unused) 3625 { 3626 complete(&ppc440spe_r6_test_comp); 3627 } 3628 3629 /** 3630 * ppc440spe_adma_issue_pending - flush all pending descriptors to h/w 3631 */ 3632 static void ppc440spe_adma_issue_pending(struct dma_chan *chan) 3633 { 3634 struct ppc440spe_adma_chan *ppc440spe_chan; 3635 3636 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 3637 dev_dbg(ppc440spe_chan->device->common.dev, 3638 "ppc440spe adma%d: %s %d\n", ppc440spe_chan->device->id, 3639 __func__, ppc440spe_chan->pending); 3640 3641 if (ppc440spe_chan->pending) { 3642 ppc440spe_chan->pending = 0; 3643 ppc440spe_chan_append(ppc440spe_chan); 3644 } 3645 } 3646 3647 /** 3648 * ppc440spe_chan_start_null_xor - initiate the first XOR operation (DMA engines 3649 * use FIFOs (as opposite to chains used in XOR) so this is a XOR 3650 * specific operation) 3651 */ 3652 static void ppc440spe_chan_start_null_xor(struct ppc440spe_adma_chan *chan) 3653 { 3654 struct ppc440spe_adma_desc_slot *sw_desc, *group_start; 3655 dma_cookie_t cookie; 3656 int slot_cnt, slots_per_op; 3657 3658 dev_dbg(chan->device->common.dev, 3659 "ppc440spe adma%d: %s\n", chan->device->id, __func__); 3660 3661 spin_lock_bh(&chan->lock); 3662 slot_cnt = ppc440spe_chan_xor_slot_count(0, 2, &slots_per_op); 3663 sw_desc = ppc440spe_adma_alloc_slots(chan, slot_cnt, slots_per_op); 3664 if (sw_desc) { 3665 group_start = sw_desc->group_head; 3666 list_splice_init(&sw_desc->group_list, &chan->chain); 3667 async_tx_ack(&sw_desc->async_tx); 3668 ppc440spe_desc_init_null_xor(group_start); 3669 3670 cookie = dma_cookie_assign(&sw_desc->async_tx); 3671 3672 /* initialize the completed cookie to be less than 3673 * the most recently used cookie 3674 */ 3675 chan->common.completed_cookie = cookie - 1; 3676 3677 /* channel should not be busy */ 3678 BUG_ON(ppc440spe_chan_is_busy(chan)); 3679 3680 /* set the descriptor address */ 3681 ppc440spe_chan_set_first_xor_descriptor(chan, sw_desc); 3682 3683 /* run the descriptor */ 3684 ppc440spe_chan_run(chan); 3685 } else 3686 printk(KERN_ERR "ppc440spe adma%d" 3687 " failed to allocate null descriptor\n", 3688 chan->device->id); 3689 spin_unlock_bh(&chan->lock); 3690 } 3691 3692 /** 3693 * ppc440spe_test_raid6 - test are RAID-6 capabilities enabled successfully. 3694 * For this we just perform one WXOR operation with the same source 3695 * and destination addresses, the GF-multiplier is 1; so if RAID-6 3696 * capabilities are enabled then we'll get src/dst filled with zero. 3697 */ 3698 static int ppc440spe_test_raid6(struct ppc440spe_adma_chan *chan) 3699 { 3700 struct ppc440spe_adma_desc_slot *sw_desc, *iter; 3701 struct page *pg; 3702 char *a; 3703 dma_addr_t dma_addr, addrs[2]; 3704 unsigned long op = 0; 3705 int rval = 0; 3706 3707 set_bit(PPC440SPE_DESC_WXOR, &op); 3708 3709 pg = alloc_page(GFP_KERNEL); 3710 if (!pg) 3711 return -ENOMEM; 3712 3713 spin_lock_bh(&chan->lock); 3714 sw_desc = ppc440spe_adma_alloc_slots(chan, 1, 1); 3715 if (sw_desc) { 3716 /* 1 src, 1 dsr, int_ena, WXOR */ 3717 ppc440spe_desc_init_dma01pq(sw_desc, 1, 1, 1, op); 3718 list_for_each_entry(iter, &sw_desc->group_list, chain_node) { 3719 ppc440spe_desc_set_byte_count(iter, chan, PAGE_SIZE); 3720 iter->unmap_len = PAGE_SIZE; 3721 } 3722 } else { 3723 rval = -EFAULT; 3724 spin_unlock_bh(&chan->lock); 3725 goto exit; 3726 } 3727 spin_unlock_bh(&chan->lock); 3728 3729 /* Fill the test page with ones */ 3730 memset(page_address(pg), 0xFF, PAGE_SIZE); 3731 dma_addr = dma_map_page(chan->device->dev, pg, 0, 3732 PAGE_SIZE, DMA_BIDIRECTIONAL); 3733 3734 /* Setup addresses */ 3735 ppc440spe_adma_pq_set_src(sw_desc, dma_addr, 0); 3736 ppc440spe_adma_pq_set_src_mult(sw_desc, 1, 0, 0); 3737 addrs[0] = dma_addr; 3738 addrs[1] = 0; 3739 ppc440spe_adma_pq_set_dest(sw_desc, addrs, DMA_PREP_PQ_DISABLE_Q); 3740 3741 async_tx_ack(&sw_desc->async_tx); 3742 sw_desc->async_tx.callback = ppc440spe_test_callback; 3743 sw_desc->async_tx.callback_param = NULL; 3744 3745 init_completion(&ppc440spe_r6_test_comp); 3746 3747 ppc440spe_adma_tx_submit(&sw_desc->async_tx); 3748 ppc440spe_adma_issue_pending(&chan->common); 3749 3750 wait_for_completion(&ppc440spe_r6_test_comp); 3751 3752 /* Now check if the test page is zeroed */ 3753 a = page_address(pg); 3754 if ((*(u32 *)a) == 0 && memcmp(a, a+4, PAGE_SIZE-4) == 0) { 3755 /* page is zero - RAID-6 enabled */ 3756 rval = 0; 3757 } else { 3758 /* RAID-6 was not enabled */ 3759 rval = -EINVAL; 3760 } 3761 exit: 3762 __free_page(pg); 3763 return rval; 3764 } 3765 3766 static void ppc440spe_adma_init_capabilities(struct ppc440spe_adma_device *adev) 3767 { 3768 switch (adev->id) { 3769 case PPC440SPE_DMA0_ID: 3770 case PPC440SPE_DMA1_ID: 3771 dma_cap_set(DMA_MEMCPY, adev->common.cap_mask); 3772 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3773 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3774 dma_cap_set(DMA_PQ_VAL, adev->common.cap_mask); 3775 dma_cap_set(DMA_XOR_VAL, adev->common.cap_mask); 3776 break; 3777 case PPC440SPE_XOR_ID: 3778 dma_cap_set(DMA_XOR, adev->common.cap_mask); 3779 dma_cap_set(DMA_PQ, adev->common.cap_mask); 3780 dma_cap_set(DMA_INTERRUPT, adev->common.cap_mask); 3781 adev->common.cap_mask = adev->common.cap_mask; 3782 break; 3783 } 3784 3785 /* Set base routines */ 3786 adev->common.device_alloc_chan_resources = 3787 ppc440spe_adma_alloc_chan_resources; 3788 adev->common.device_free_chan_resources = 3789 ppc440spe_adma_free_chan_resources; 3790 adev->common.device_tx_status = ppc440spe_adma_tx_status; 3791 adev->common.device_issue_pending = ppc440spe_adma_issue_pending; 3792 3793 /* Set prep routines based on capability */ 3794 if (dma_has_cap(DMA_MEMCPY, adev->common.cap_mask)) { 3795 adev->common.device_prep_dma_memcpy = 3796 ppc440spe_adma_prep_dma_memcpy; 3797 } 3798 if (dma_has_cap(DMA_XOR, adev->common.cap_mask)) { 3799 adev->common.max_xor = XOR_MAX_OPS; 3800 adev->common.device_prep_dma_xor = 3801 ppc440spe_adma_prep_dma_xor; 3802 } 3803 if (dma_has_cap(DMA_PQ, adev->common.cap_mask)) { 3804 switch (adev->id) { 3805 case PPC440SPE_DMA0_ID: 3806 dma_set_maxpq(&adev->common, 3807 DMA0_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3808 break; 3809 case PPC440SPE_DMA1_ID: 3810 dma_set_maxpq(&adev->common, 3811 DMA1_FIFO_SIZE / sizeof(struct dma_cdb), 0); 3812 break; 3813 case PPC440SPE_XOR_ID: 3814 adev->common.max_pq = XOR_MAX_OPS * 3; 3815 break; 3816 } 3817 adev->common.device_prep_dma_pq = 3818 ppc440spe_adma_prep_dma_pq; 3819 } 3820 if (dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask)) { 3821 switch (adev->id) { 3822 case PPC440SPE_DMA0_ID: 3823 adev->common.max_pq = DMA0_FIFO_SIZE / 3824 sizeof(struct dma_cdb); 3825 break; 3826 case PPC440SPE_DMA1_ID: 3827 adev->common.max_pq = DMA1_FIFO_SIZE / 3828 sizeof(struct dma_cdb); 3829 break; 3830 } 3831 adev->common.device_prep_dma_pq_val = 3832 ppc440spe_adma_prep_dma_pqzero_sum; 3833 } 3834 if (dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask)) { 3835 switch (adev->id) { 3836 case PPC440SPE_DMA0_ID: 3837 adev->common.max_xor = DMA0_FIFO_SIZE / 3838 sizeof(struct dma_cdb); 3839 break; 3840 case PPC440SPE_DMA1_ID: 3841 adev->common.max_xor = DMA1_FIFO_SIZE / 3842 sizeof(struct dma_cdb); 3843 break; 3844 } 3845 adev->common.device_prep_dma_xor_val = 3846 ppc440spe_adma_prep_dma_xor_zero_sum; 3847 } 3848 if (dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask)) { 3849 adev->common.device_prep_dma_interrupt = 3850 ppc440spe_adma_prep_dma_interrupt; 3851 } 3852 pr_info("%s: AMCC(R) PPC440SP(E) ADMA Engine: " 3853 "( %s%s%s%s%s%s)\n", 3854 dev_name(adev->dev), 3855 dma_has_cap(DMA_PQ, adev->common.cap_mask) ? "pq " : "", 3856 dma_has_cap(DMA_PQ_VAL, adev->common.cap_mask) ? "pq_val " : "", 3857 dma_has_cap(DMA_XOR, adev->common.cap_mask) ? "xor " : "", 3858 dma_has_cap(DMA_XOR_VAL, adev->common.cap_mask) ? "xor_val " : "", 3859 dma_has_cap(DMA_MEMCPY, adev->common.cap_mask) ? "memcpy " : "", 3860 dma_has_cap(DMA_INTERRUPT, adev->common.cap_mask) ? "intr " : ""); 3861 } 3862 3863 static int ppc440spe_adma_setup_irqs(struct ppc440spe_adma_device *adev, 3864 struct ppc440spe_adma_chan *chan, 3865 int *initcode) 3866 { 3867 struct platform_device *ofdev; 3868 struct device_node *np; 3869 int ret; 3870 3871 ofdev = container_of(adev->dev, struct platform_device, dev); 3872 np = ofdev->dev.of_node; 3873 if (adev->id != PPC440SPE_XOR_ID) { 3874 adev->err_irq = irq_of_parse_and_map(np, 1); 3875 if (!adev->err_irq) { 3876 dev_warn(adev->dev, "no err irq resource?\n"); 3877 *initcode = PPC_ADMA_INIT_IRQ2; 3878 adev->err_irq = -ENXIO; 3879 } else 3880 atomic_inc(&ppc440spe_adma_err_irq_ref); 3881 } else { 3882 adev->err_irq = -ENXIO; 3883 } 3884 3885 adev->irq = irq_of_parse_and_map(np, 0); 3886 if (!adev->irq) { 3887 dev_err(adev->dev, "no irq resource\n"); 3888 *initcode = PPC_ADMA_INIT_IRQ1; 3889 ret = -ENXIO; 3890 goto err_irq_map; 3891 } 3892 dev_dbg(adev->dev, "irq %d, err irq %d\n", 3893 adev->irq, adev->err_irq); 3894 3895 ret = request_irq(adev->irq, ppc440spe_adma_eot_handler, 3896 0, dev_driver_string(adev->dev), chan); 3897 if (ret) { 3898 dev_err(adev->dev, "can't request irq %d\n", 3899 adev->irq); 3900 *initcode = PPC_ADMA_INIT_IRQ1; 3901 ret = -EIO; 3902 goto err_req1; 3903 } 3904 3905 /* only DMA engines have a separate error IRQ 3906 * so it's Ok if err_irq < 0 in XOR engine case. 3907 */ 3908 if (adev->err_irq > 0) { 3909 /* both DMA engines share common error IRQ */ 3910 ret = request_irq(adev->err_irq, 3911 ppc440spe_adma_err_handler, 3912 IRQF_SHARED, 3913 dev_driver_string(adev->dev), 3914 chan); 3915 if (ret) { 3916 dev_err(adev->dev, "can't request irq %d\n", 3917 adev->err_irq); 3918 *initcode = PPC_ADMA_INIT_IRQ2; 3919 ret = -EIO; 3920 goto err_req2; 3921 } 3922 } 3923 3924 if (adev->id == PPC440SPE_XOR_ID) { 3925 /* enable XOR engine interrupts */ 3926 iowrite32be(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3927 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT, 3928 &adev->xor_reg->ier); 3929 } else { 3930 u32 mask, enable; 3931 3932 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 3933 if (!np) { 3934 pr_err("%s: can't find I2O device tree node\n", 3935 __func__); 3936 ret = -ENODEV; 3937 goto err_req2; 3938 } 3939 adev->i2o_reg = of_iomap(np, 0); 3940 if (!adev->i2o_reg) { 3941 pr_err("%s: failed to map I2O registers\n", __func__); 3942 of_node_put(np); 3943 ret = -EINVAL; 3944 goto err_req2; 3945 } 3946 of_node_put(np); 3947 /* Unmask 'CS FIFO Attention' interrupts and 3948 * enable generating interrupts on errors 3949 */ 3950 enable = (adev->id == PPC440SPE_DMA0_ID) ? 3951 ~(I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3952 ~(I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3953 mask = ioread32(&adev->i2o_reg->iopim) & enable; 3954 iowrite32(mask, &adev->i2o_reg->iopim); 3955 } 3956 return 0; 3957 3958 err_req2: 3959 free_irq(adev->irq, chan); 3960 err_req1: 3961 irq_dispose_mapping(adev->irq); 3962 err_irq_map: 3963 if (adev->err_irq > 0) { 3964 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) 3965 irq_dispose_mapping(adev->err_irq); 3966 } 3967 return ret; 3968 } 3969 3970 static void ppc440spe_adma_release_irqs(struct ppc440spe_adma_device *adev, 3971 struct ppc440spe_adma_chan *chan) 3972 { 3973 u32 mask, disable; 3974 3975 if (adev->id == PPC440SPE_XOR_ID) { 3976 /* disable XOR engine interrupts */ 3977 mask = ioread32be(&adev->xor_reg->ier); 3978 mask &= ~(XOR_IE_CBCIE_BIT | XOR_IE_ICBIE_BIT | 3979 XOR_IE_ICIE_BIT | XOR_IE_RPTIE_BIT); 3980 iowrite32be(mask, &adev->xor_reg->ier); 3981 } else { 3982 /* disable DMAx engine interrupts */ 3983 disable = (adev->id == PPC440SPE_DMA0_ID) ? 3984 (I2O_IOPIM_P0SNE | I2O_IOPIM_P0EM) : 3985 (I2O_IOPIM_P1SNE | I2O_IOPIM_P1EM); 3986 mask = ioread32(&adev->i2o_reg->iopim) | disable; 3987 iowrite32(mask, &adev->i2o_reg->iopim); 3988 } 3989 free_irq(adev->irq, chan); 3990 irq_dispose_mapping(adev->irq); 3991 if (adev->err_irq > 0) { 3992 free_irq(adev->err_irq, chan); 3993 if (atomic_dec_and_test(&ppc440spe_adma_err_irq_ref)) { 3994 irq_dispose_mapping(adev->err_irq); 3995 iounmap(adev->i2o_reg); 3996 } 3997 } 3998 } 3999 4000 /** 4001 * ppc440spe_adma_probe - probe the asynch device 4002 */ 4003 static int ppc440spe_adma_probe(struct platform_device *ofdev) 4004 { 4005 struct device_node *np = ofdev->dev.of_node; 4006 struct resource res; 4007 struct ppc440spe_adma_device *adev; 4008 struct ppc440spe_adma_chan *chan; 4009 struct ppc_dma_chan_ref *ref, *_ref; 4010 int ret = 0, initcode = PPC_ADMA_INIT_OK; 4011 const u32 *idx; 4012 int len; 4013 void *regs; 4014 u32 id, pool_size; 4015 4016 if (of_device_is_compatible(np, "amcc,xor-accelerator")) { 4017 id = PPC440SPE_XOR_ID; 4018 /* As far as the XOR engine is concerned, it does not 4019 * use FIFOs but uses linked list. So there is no dependency 4020 * between pool size to allocate and the engine configuration. 4021 */ 4022 pool_size = PAGE_SIZE << 1; 4023 } else { 4024 /* it is DMA0 or DMA1 */ 4025 idx = of_get_property(np, "cell-index", &len); 4026 if (!idx || (len != sizeof(u32))) { 4027 dev_err(&ofdev->dev, "Device node %pOF has missing " 4028 "or invalid cell-index property\n", 4029 np); 4030 return -EINVAL; 4031 } 4032 id = *idx; 4033 /* DMA0,1 engines use FIFO to maintain CDBs, so we 4034 * should allocate the pool accordingly to size of this 4035 * FIFO. Thus, the pool size depends on the FIFO depth: 4036 * how much CDBs pointers the FIFO may contain then so 4037 * much CDBs we should provide in the pool. 4038 * That is 4039 * CDB size = 32B; 4040 * CDBs number = (DMA0_FIFO_SIZE >> 3); 4041 * Pool size = CDBs number * CDB size = 4042 * = (DMA0_FIFO_SIZE >> 3) << 5 = DMA0_FIFO_SIZE << 2. 4043 */ 4044 pool_size = (id == PPC440SPE_DMA0_ID) ? 4045 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4046 pool_size <<= 2; 4047 } 4048 4049 if (of_address_to_resource(np, 0, &res)) { 4050 dev_err(&ofdev->dev, "failed to get memory resource\n"); 4051 initcode = PPC_ADMA_INIT_MEMRES; 4052 ret = -ENODEV; 4053 goto out; 4054 } 4055 4056 if (!request_mem_region(res.start, resource_size(&res), 4057 dev_driver_string(&ofdev->dev))) { 4058 dev_err(&ofdev->dev, "failed to request memory region %pR\n", 4059 &res); 4060 initcode = PPC_ADMA_INIT_MEMREG; 4061 ret = -EBUSY; 4062 goto out; 4063 } 4064 4065 /* create a device */ 4066 adev = kzalloc_obj(*adev); 4067 if (!adev) { 4068 initcode = PPC_ADMA_INIT_ALLOC; 4069 ret = -ENOMEM; 4070 goto err_adev_alloc; 4071 } 4072 4073 adev->id = id; 4074 adev->pool_size = pool_size; 4075 /* allocate coherent memory for hardware descriptors */ 4076 adev->dma_desc_pool_virt = dma_alloc_coherent(&ofdev->dev, 4077 adev->pool_size, &adev->dma_desc_pool, 4078 GFP_KERNEL); 4079 if (adev->dma_desc_pool_virt == NULL) { 4080 dev_err(&ofdev->dev, "failed to allocate %d bytes of coherent " 4081 "memory for hardware descriptors\n", 4082 adev->pool_size); 4083 initcode = PPC_ADMA_INIT_COHERENT; 4084 ret = -ENOMEM; 4085 goto err_dma_alloc; 4086 } 4087 dev_dbg(&ofdev->dev, "allocated descriptor pool virt 0x%p phys 0x%llx\n", 4088 adev->dma_desc_pool_virt, (u64)adev->dma_desc_pool); 4089 4090 regs = ioremap(res.start, resource_size(&res)); 4091 if (!regs) { 4092 dev_err(&ofdev->dev, "failed to ioremap regs!\n"); 4093 ret = -ENOMEM; 4094 goto err_regs_alloc; 4095 } 4096 4097 if (adev->id == PPC440SPE_XOR_ID) { 4098 adev->xor_reg = regs; 4099 /* Reset XOR */ 4100 iowrite32be(XOR_CRSR_XASR_BIT, &adev->xor_reg->crsr); 4101 iowrite32be(XOR_CRSR_64BA_BIT, &adev->xor_reg->crrr); 4102 } else { 4103 size_t fifo_size = (adev->id == PPC440SPE_DMA0_ID) ? 4104 DMA0_FIFO_SIZE : DMA1_FIFO_SIZE; 4105 adev->dma_reg = regs; 4106 /* DMAx_FIFO_SIZE is defined in bytes, 4107 * <fsiz> - is defined in number of CDB pointers (8byte). 4108 * DMA FIFO Length = CSlength + CPlength, where 4109 * CSlength = CPlength = (fsiz + 1) * 8. 4110 */ 4111 iowrite32(DMA_FIFO_ENABLE | ((fifo_size >> 3) - 2), 4112 &adev->dma_reg->fsiz); 4113 /* Configure DMA engine */ 4114 iowrite32(DMA_CFG_DXEPR_HP | DMA_CFG_DFMPP_HP | DMA_CFG_FALGN, 4115 &adev->dma_reg->cfg); 4116 /* Clear Status */ 4117 iowrite32(~0, &adev->dma_reg->dsts); 4118 } 4119 4120 adev->dev = &ofdev->dev; 4121 adev->common.dev = &ofdev->dev; 4122 INIT_LIST_HEAD(&adev->common.channels); 4123 platform_set_drvdata(ofdev, adev); 4124 4125 /* create a channel */ 4126 chan = kzalloc_obj(*chan); 4127 if (!chan) { 4128 initcode = PPC_ADMA_INIT_CHANNEL; 4129 ret = -ENOMEM; 4130 goto err_chan_alloc; 4131 } 4132 4133 spin_lock_init(&chan->lock); 4134 INIT_LIST_HEAD(&chan->chain); 4135 INIT_LIST_HEAD(&chan->all_slots); 4136 chan->device = adev; 4137 chan->common.device = &adev->common; 4138 dma_cookie_init(&chan->common); 4139 list_add_tail(&chan->common.device_node, &adev->common.channels); 4140 tasklet_setup(&chan->irq_tasklet, ppc440spe_adma_tasklet); 4141 4142 /* allocate and map helper pages for async validation or 4143 * async_mult/async_sum_product operations on DMA0/1. 4144 */ 4145 if (adev->id != PPC440SPE_XOR_ID) { 4146 chan->pdest_page = alloc_page(GFP_KERNEL); 4147 chan->qdest_page = alloc_page(GFP_KERNEL); 4148 if (!chan->pdest_page || 4149 !chan->qdest_page) { 4150 if (chan->pdest_page) 4151 __free_page(chan->pdest_page); 4152 if (chan->qdest_page) 4153 __free_page(chan->qdest_page); 4154 ret = -ENOMEM; 4155 goto err_page_alloc; 4156 } 4157 chan->pdest = dma_map_page(&ofdev->dev, chan->pdest_page, 0, 4158 PAGE_SIZE, DMA_BIDIRECTIONAL); 4159 chan->qdest = dma_map_page(&ofdev->dev, chan->qdest_page, 0, 4160 PAGE_SIZE, DMA_BIDIRECTIONAL); 4161 } 4162 4163 ref = kmalloc_obj(*ref); 4164 if (ref) { 4165 ref->chan = &chan->common; 4166 INIT_LIST_HEAD(&ref->node); 4167 list_add_tail(&ref->node, &ppc440spe_adma_chan_list); 4168 } else { 4169 dev_err(&ofdev->dev, "failed to allocate channel reference!\n"); 4170 ret = -ENOMEM; 4171 goto err_ref_alloc; 4172 } 4173 4174 ret = ppc440spe_adma_setup_irqs(adev, chan, &initcode); 4175 if (ret) 4176 goto err_irq; 4177 4178 ppc440spe_adma_init_capabilities(adev); 4179 4180 ret = dma_async_device_register(&adev->common); 4181 if (ret) { 4182 initcode = PPC_ADMA_INIT_REGISTER; 4183 dev_err(&ofdev->dev, "failed to register dma device\n"); 4184 goto err_dev_reg; 4185 } 4186 4187 goto out; 4188 4189 err_dev_reg: 4190 ppc440spe_adma_release_irqs(adev, chan); 4191 err_irq: 4192 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, node) { 4193 if (chan == to_ppc440spe_adma_chan(ref->chan)) { 4194 list_del(&ref->node); 4195 kfree(ref); 4196 } 4197 } 4198 err_ref_alloc: 4199 if (adev->id != PPC440SPE_XOR_ID) { 4200 dma_unmap_page(&ofdev->dev, chan->pdest, 4201 PAGE_SIZE, DMA_BIDIRECTIONAL); 4202 dma_unmap_page(&ofdev->dev, chan->qdest, 4203 PAGE_SIZE, DMA_BIDIRECTIONAL); 4204 __free_page(chan->pdest_page); 4205 __free_page(chan->qdest_page); 4206 } 4207 err_page_alloc: 4208 kfree(chan); 4209 err_chan_alloc: 4210 if (adev->id == PPC440SPE_XOR_ID) 4211 iounmap(adev->xor_reg); 4212 else 4213 iounmap(adev->dma_reg); 4214 err_regs_alloc: 4215 dma_free_coherent(adev->dev, adev->pool_size, 4216 adev->dma_desc_pool_virt, 4217 adev->dma_desc_pool); 4218 err_dma_alloc: 4219 kfree(adev); 4220 err_adev_alloc: 4221 release_mem_region(res.start, resource_size(&res)); 4222 out: 4223 if (id < PPC440SPE_ADMA_ENGINES_NUM) 4224 ppc440spe_adma_devices[id] = initcode; 4225 4226 return ret; 4227 } 4228 4229 /** 4230 * ppc440spe_adma_remove - remove the asynch device 4231 */ 4232 static void ppc440spe_adma_remove(struct platform_device *ofdev) 4233 { 4234 struct ppc440spe_adma_device *adev = platform_get_drvdata(ofdev); 4235 struct device_node *np = ofdev->dev.of_node; 4236 struct resource res; 4237 struct dma_chan *chan, *_chan; 4238 struct ppc_dma_chan_ref *ref, *_ref; 4239 struct ppc440spe_adma_chan *ppc440spe_chan; 4240 4241 if (adev->id < PPC440SPE_ADMA_ENGINES_NUM) 4242 ppc440spe_adma_devices[adev->id] = -1; 4243 4244 dma_async_device_unregister(&adev->common); 4245 4246 list_for_each_entry_safe(chan, _chan, &adev->common.channels, 4247 device_node) { 4248 ppc440spe_chan = to_ppc440spe_adma_chan(chan); 4249 ppc440spe_adma_release_irqs(adev, ppc440spe_chan); 4250 tasklet_kill(&ppc440spe_chan->irq_tasklet); 4251 if (adev->id != PPC440SPE_XOR_ID) { 4252 dma_unmap_page(&ofdev->dev, ppc440spe_chan->pdest, 4253 PAGE_SIZE, DMA_BIDIRECTIONAL); 4254 dma_unmap_page(&ofdev->dev, ppc440spe_chan->qdest, 4255 PAGE_SIZE, DMA_BIDIRECTIONAL); 4256 __free_page(ppc440spe_chan->pdest_page); 4257 __free_page(ppc440spe_chan->qdest_page); 4258 } 4259 list_for_each_entry_safe(ref, _ref, &ppc440spe_adma_chan_list, 4260 node) { 4261 if (ppc440spe_chan == 4262 to_ppc440spe_adma_chan(ref->chan)) { 4263 list_del(&ref->node); 4264 kfree(ref); 4265 } 4266 } 4267 list_del(&chan->device_node); 4268 kfree(ppc440spe_chan); 4269 } 4270 4271 dma_free_coherent(adev->dev, adev->pool_size, 4272 adev->dma_desc_pool_virt, adev->dma_desc_pool); 4273 if (adev->id == PPC440SPE_XOR_ID) 4274 iounmap(adev->xor_reg); 4275 else 4276 iounmap(adev->dma_reg); 4277 of_address_to_resource(np, 0, &res); 4278 release_mem_region(res.start, resource_size(&res)); 4279 kfree(adev); 4280 } 4281 4282 /* 4283 * /sys driver interface to enable h/w RAID-6 capabilities 4284 * Files created in e.g. /sys/devices/plb.0/400100100.dma0/driver/ 4285 * directory are "devices", "enable" and "poly". 4286 * "devices" shows available engines. 4287 * "enable" is used to enable RAID-6 capabilities or to check 4288 * whether these has been activated. 4289 * "poly" allows setting/checking used polynomial (for PPC440SPe only). 4290 */ 4291 4292 static ssize_t devices_show(struct device_driver *dev, char *buf) 4293 { 4294 ssize_t size = 0; 4295 int i; 4296 4297 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) { 4298 if (ppc440spe_adma_devices[i] == -1) 4299 continue; 4300 size += sysfs_emit_at(buf, size, "PPC440SP(E)-ADMA.%d: %s\n", 4301 i, ppc_adma_errors[ppc440spe_adma_devices[i]]); 4302 } 4303 return size; 4304 } 4305 static DRIVER_ATTR_RO(devices); 4306 4307 static ssize_t enable_show(struct device_driver *dev, char *buf) 4308 { 4309 return sysfs_emit(buf, "PPC440SP(e) RAID-6 capabilities are %sABLED.\n", 4310 ppc440spe_r6_enabled ? "EN" : "DIS"); 4311 } 4312 4313 static ssize_t enable_store(struct device_driver *dev, const char *buf, 4314 size_t count) 4315 { 4316 unsigned long val; 4317 int err; 4318 4319 if (!count || count > 11) 4320 return -EINVAL; 4321 4322 if (!ppc440spe_r6_tchan) 4323 return -EFAULT; 4324 4325 /* Write a key */ 4326 err = kstrtoul(buf, 16, &val); 4327 if (err) 4328 return err; 4329 4330 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_XORBA, val); 4331 isync(); 4332 4333 /* Verify whether it really works now */ 4334 if (ppc440spe_test_raid6(ppc440spe_r6_tchan) == 0) { 4335 pr_info("PPC440SP(e) RAID-6 has been activated " 4336 "successfully\n"); 4337 ppc440spe_r6_enabled = 1; 4338 } else { 4339 pr_info("PPC440SP(e) RAID-6 hasn't been activated!" 4340 " Error key ?\n"); 4341 ppc440spe_r6_enabled = 0; 4342 } 4343 return count; 4344 } 4345 static DRIVER_ATTR_RW(enable); 4346 4347 static ssize_t poly_show(struct device_driver *dev, char *buf) 4348 { 4349 ssize_t size = 0; 4350 u32 reg; 4351 4352 #ifdef CONFIG_440SP 4353 /* 440SP has fixed polynomial */ 4354 reg = 0x4d; 4355 #else 4356 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4357 reg >>= MQ0_CFBHL_POLY; 4358 reg &= 0xFF; 4359 #endif 4360 4361 size = sysfs_emit(buf, "PPC440SP(e) RAID-6 driver " 4362 "uses 0x1%02x polynomial.\n", reg); 4363 return size; 4364 } 4365 4366 static ssize_t poly_store(struct device_driver *dev, const char *buf, 4367 size_t count) 4368 { 4369 unsigned long reg, val; 4370 int err; 4371 #ifdef CONFIG_440SP 4372 /* 440SP uses default 0x14D polynomial only */ 4373 return -EINVAL; 4374 #endif 4375 4376 if (!count || count > 6) 4377 return -EINVAL; 4378 4379 /* e.g., 0x14D or 0x11D */ 4380 err = kstrtoul(buf, 16, &val); 4381 if (err) 4382 return err; 4383 4384 if (val & ~0x1FF) 4385 return -EINVAL; 4386 4387 val &= 0xFF; 4388 reg = dcr_read(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL); 4389 reg &= ~(0xFF << MQ0_CFBHL_POLY); 4390 reg |= val << MQ0_CFBHL_POLY; 4391 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, reg); 4392 4393 return count; 4394 } 4395 static DRIVER_ATTR_RW(poly); 4396 4397 /* 4398 * Common initialisation for RAID engines; allocate memory for 4399 * DMAx FIFOs, perform configuration common for all DMA engines. 4400 * Further DMA engine specific configuration is done at probe time. 4401 */ 4402 static int ppc440spe_configure_raid_devices(void) 4403 { 4404 struct device_node *np; 4405 struct resource i2o_res; 4406 struct i2o_regs __iomem *i2o_reg; 4407 dcr_host_t i2o_dcr_host; 4408 unsigned int dcr_base, dcr_len; 4409 int i, ret; 4410 4411 np = of_find_compatible_node(NULL, NULL, "ibm,i2o-440spe"); 4412 if (!np) { 4413 pr_err("%s: can't find I2O device tree node\n", 4414 __func__); 4415 return -ENODEV; 4416 } 4417 4418 if (of_address_to_resource(np, 0, &i2o_res)) { 4419 of_node_put(np); 4420 return -EINVAL; 4421 } 4422 4423 i2o_reg = of_iomap(np, 0); 4424 if (!i2o_reg) { 4425 pr_err("%s: failed to map I2O registers\n", __func__); 4426 of_node_put(np); 4427 return -EINVAL; 4428 } 4429 4430 /* Get I2O DCRs base */ 4431 dcr_base = dcr_resource_start(np, 0); 4432 dcr_len = dcr_resource_len(np, 0); 4433 if (!dcr_base && !dcr_len) { 4434 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4435 of_node_put(np); 4436 iounmap(i2o_reg); 4437 return -ENODEV; 4438 } 4439 4440 i2o_dcr_host = dcr_map(np, dcr_base, dcr_len); 4441 if (!DCR_MAP_OK(i2o_dcr_host)) { 4442 pr_err("%pOF: failed to map DCRs!\n", np); 4443 of_node_put(np); 4444 iounmap(i2o_reg); 4445 return -ENODEV; 4446 } 4447 of_node_put(np); 4448 4449 /* Provide memory regions for DMA's FIFOs: I2O, DMA0 and DMA1 share 4450 * the base address of FIFO memory space. 4451 * Actually we need twice more physical memory than programmed in the 4452 * <fsiz> register (because there are two FIFOs for each DMA: CP and CS) 4453 */ 4454 ppc440spe_dma_fifo_buf = kmalloc((DMA0_FIFO_SIZE + DMA1_FIFO_SIZE) << 1, 4455 GFP_KERNEL); 4456 if (!ppc440spe_dma_fifo_buf) { 4457 pr_err("%s: DMA FIFO buffer allocation failed.\n", __func__); 4458 iounmap(i2o_reg); 4459 dcr_unmap(i2o_dcr_host, dcr_len); 4460 return -ENOMEM; 4461 } 4462 4463 /* 4464 * Configure h/w 4465 */ 4466 /* Reset I2O/DMA */ 4467 mtdcri(SDR0, DCRN_SDR0_SRST, DCRN_SDR0_SRST_I2ODMA); 4468 mtdcri(SDR0, DCRN_SDR0_SRST, 0); 4469 4470 /* Setup the base address of mmaped registers */ 4471 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAH, (u32)(i2o_res.start >> 32)); 4472 dcr_write(i2o_dcr_host, DCRN_I2O0_IBAL, (u32)(i2o_res.start) | 4473 I2O_REG_ENABLE); 4474 dcr_unmap(i2o_dcr_host, dcr_len); 4475 4476 /* Setup FIFO memory space base address */ 4477 iowrite32(0, &i2o_reg->ifbah); 4478 iowrite32(((u32)__pa(ppc440spe_dma_fifo_buf)), &i2o_reg->ifbal); 4479 4480 /* set zero FIFO size for I2O, so the whole 4481 * ppc440spe_dma_fifo_buf is used by DMAs. 4482 * DMAx_FIFOs will be configured while probe. 4483 */ 4484 iowrite32(0, &i2o_reg->ifsiz); 4485 iounmap(i2o_reg); 4486 4487 /* To prepare WXOR/RXOR functionality we need access to 4488 * Memory Queue Module DCRs (finally it will be enabled 4489 * via /sys interface of the ppc440spe ADMA driver). 4490 */ 4491 np = of_find_compatible_node(NULL, NULL, "ibm,mq-440spe"); 4492 if (!np) { 4493 pr_err("%s: can't find MQ device tree node\n", 4494 __func__); 4495 ret = -ENODEV; 4496 goto out_free; 4497 } 4498 4499 /* Get MQ DCRs base */ 4500 dcr_base = dcr_resource_start(np, 0); 4501 dcr_len = dcr_resource_len(np, 0); 4502 if (!dcr_base && !dcr_len) { 4503 pr_err("%pOF: can't get DCR registers base/len!\n", np); 4504 ret = -ENODEV; 4505 goto out_mq; 4506 } 4507 4508 ppc440spe_mq_dcr_host = dcr_map(np, dcr_base, dcr_len); 4509 if (!DCR_MAP_OK(ppc440spe_mq_dcr_host)) { 4510 pr_err("%pOF: failed to map DCRs!\n", np); 4511 ret = -ENODEV; 4512 goto out_mq; 4513 } 4514 of_node_put(np); 4515 ppc440spe_mq_dcr_len = dcr_len; 4516 4517 /* Set HB alias */ 4518 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_BAUH, DMA_CUED_XOR_HB); 4519 4520 /* Set: 4521 * - LL transaction passing limit to 1; 4522 * - Memory controller cycle limit to 1; 4523 * - Galois Polynomial to 0x14d (default) 4524 */ 4525 dcr_write(ppc440spe_mq_dcr_host, DCRN_MQ0_CFBHL, 4526 (1 << MQ0_CFBHL_TPLM) | (1 << MQ0_CFBHL_HBCL) | 4527 (PPC440SPE_DEFAULT_POLY << MQ0_CFBHL_POLY)); 4528 4529 atomic_set(&ppc440spe_adma_err_irq_ref, 0); 4530 for (i = 0; i < PPC440SPE_ADMA_ENGINES_NUM; i++) 4531 ppc440spe_adma_devices[i] = -1; 4532 4533 return 0; 4534 4535 out_mq: 4536 of_node_put(np); 4537 out_free: 4538 kfree(ppc440spe_dma_fifo_buf); 4539 return ret; 4540 } 4541 4542 static const struct of_device_id ppc440spe_adma_of_match[] = { 4543 { .compatible = "ibm,dma-440spe", }, 4544 { .compatible = "amcc,xor-accelerator", }, 4545 {}, 4546 }; 4547 MODULE_DEVICE_TABLE(of, ppc440spe_adma_of_match); 4548 4549 static struct platform_driver ppc440spe_adma_driver = { 4550 .probe = ppc440spe_adma_probe, 4551 .remove = ppc440spe_adma_remove, 4552 .driver = { 4553 .name = "PPC440SP(E)-ADMA", 4554 .of_match_table = ppc440spe_adma_of_match, 4555 }, 4556 }; 4557 4558 static __init int ppc440spe_adma_init(void) 4559 { 4560 int ret; 4561 4562 ret = ppc440spe_configure_raid_devices(); 4563 if (ret) 4564 return ret; 4565 4566 ret = platform_driver_register(&ppc440spe_adma_driver); 4567 if (ret) { 4568 pr_err("%s: failed to register platform driver\n", 4569 __func__); 4570 goto out_reg; 4571 } 4572 4573 /* Initialization status */ 4574 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4575 &driver_attr_devices); 4576 if (ret) 4577 goto out_dev; 4578 4579 /* RAID-6 h/w enable entry */ 4580 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4581 &driver_attr_enable); 4582 if (ret) 4583 goto out_en; 4584 4585 /* GF polynomial to use */ 4586 ret = driver_create_file(&ppc440spe_adma_driver.driver, 4587 &driver_attr_poly); 4588 if (!ret) 4589 return ret; 4590 4591 driver_remove_file(&ppc440spe_adma_driver.driver, 4592 &driver_attr_enable); 4593 out_en: 4594 driver_remove_file(&ppc440spe_adma_driver.driver, 4595 &driver_attr_devices); 4596 out_dev: 4597 /* User will not be able to enable h/w RAID-6 */ 4598 pr_err("%s: failed to create RAID-6 driver interface\n", 4599 __func__); 4600 platform_driver_unregister(&ppc440spe_adma_driver); 4601 out_reg: 4602 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4603 kfree(ppc440spe_dma_fifo_buf); 4604 return ret; 4605 } 4606 4607 static void __exit ppc440spe_adma_exit(void) 4608 { 4609 driver_remove_file(&ppc440spe_adma_driver.driver, 4610 &driver_attr_poly); 4611 driver_remove_file(&ppc440spe_adma_driver.driver, 4612 &driver_attr_enable); 4613 driver_remove_file(&ppc440spe_adma_driver.driver, 4614 &driver_attr_devices); 4615 platform_driver_unregister(&ppc440spe_adma_driver); 4616 dcr_unmap(ppc440spe_mq_dcr_host, ppc440spe_mq_dcr_len); 4617 kfree(ppc440spe_dma_fifo_buf); 4618 } 4619 4620 arch_initcall(ppc440spe_adma_init); 4621 module_exit(ppc440spe_adma_exit); 4622 4623 MODULE_AUTHOR("Yuri Tikhonov <yur@emcraft.com>"); 4624 MODULE_DESCRIPTION("PPC440SPE ADMA Engine Driver"); 4625 MODULE_LICENSE("GPL"); 4626