1 /* 2 * Copyright (c) 2012 Samsung Electronics Co., Ltd. 3 * http://www.samsung.com 4 * 5 * Copyright (C) 2010 Samsung Electronics Co. Ltd. 6 * Jaswinder Singh <jassi.brar@samsung.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 */ 13 14 #include <linux/kernel.h> 15 #include <linux/io.h> 16 #include <linux/init.h> 17 #include <linux/slab.h> 18 #include <linux/module.h> 19 #include <linux/string.h> 20 #include <linux/delay.h> 21 #include <linux/interrupt.h> 22 #include <linux/dma-mapping.h> 23 #include <linux/dmaengine.h> 24 #include <linux/amba/bus.h> 25 #include <linux/amba/pl330.h> 26 #include <linux/pm_runtime.h> 27 #include <linux/scatterlist.h> 28 #include <linux/of.h> 29 30 #include "dmaengine.h" 31 #define PL330_MAX_CHAN 8 32 #define PL330_MAX_IRQS 32 33 #define PL330_MAX_PERI 32 34 35 enum pl330_srccachectrl { 36 SCCTRL0, /* Noncacheable and nonbufferable */ 37 SCCTRL1, /* Bufferable only */ 38 SCCTRL2, /* Cacheable, but do not allocate */ 39 SCCTRL3, /* Cacheable and bufferable, but do not allocate */ 40 SINVALID1, 41 SINVALID2, 42 SCCTRL6, /* Cacheable write-through, allocate on reads only */ 43 SCCTRL7, /* Cacheable write-back, allocate on reads only */ 44 }; 45 46 enum pl330_dstcachectrl { 47 DCCTRL0, /* Noncacheable and nonbufferable */ 48 DCCTRL1, /* Bufferable only */ 49 DCCTRL2, /* Cacheable, but do not allocate */ 50 DCCTRL3, /* Cacheable and bufferable, but do not allocate */ 51 DINVALID1, /* AWCACHE = 0x1000 */ 52 DINVALID2, 53 DCCTRL6, /* Cacheable write-through, allocate on writes only */ 54 DCCTRL7, /* Cacheable write-back, allocate on writes only */ 55 }; 56 57 enum pl330_byteswap { 58 SWAP_NO, 59 SWAP_2, 60 SWAP_4, 61 SWAP_8, 62 SWAP_16, 63 }; 64 65 enum pl330_reqtype { 66 MEMTOMEM, 67 MEMTODEV, 68 DEVTOMEM, 69 DEVTODEV, 70 }; 71 72 /* Register and Bit field Definitions */ 73 #define DS 0x0 74 #define DS_ST_STOP 0x0 75 #define DS_ST_EXEC 0x1 76 #define DS_ST_CMISS 0x2 77 #define DS_ST_UPDTPC 0x3 78 #define DS_ST_WFE 0x4 79 #define DS_ST_ATBRR 0x5 80 #define DS_ST_QBUSY 0x6 81 #define DS_ST_WFP 0x7 82 #define DS_ST_KILL 0x8 83 #define DS_ST_CMPLT 0x9 84 #define DS_ST_FLTCMP 0xe 85 #define DS_ST_FAULT 0xf 86 87 #define DPC 0x4 88 #define INTEN 0x20 89 #define ES 0x24 90 #define INTSTATUS 0x28 91 #define INTCLR 0x2c 92 #define FSM 0x30 93 #define FSC 0x34 94 #define FTM 0x38 95 96 #define _FTC 0x40 97 #define FTC(n) (_FTC + (n)*0x4) 98 99 #define _CS 0x100 100 #define CS(n) (_CS + (n)*0x8) 101 #define CS_CNS (1 << 21) 102 103 #define _CPC 0x104 104 #define CPC(n) (_CPC + (n)*0x8) 105 106 #define _SA 0x400 107 #define SA(n) (_SA + (n)*0x20) 108 109 #define _DA 0x404 110 #define DA(n) (_DA + (n)*0x20) 111 112 #define _CC 0x408 113 #define CC(n) (_CC + (n)*0x20) 114 115 #define CC_SRCINC (1 << 0) 116 #define CC_DSTINC (1 << 14) 117 #define CC_SRCPRI (1 << 8) 118 #define CC_DSTPRI (1 << 22) 119 #define CC_SRCNS (1 << 9) 120 #define CC_DSTNS (1 << 23) 121 #define CC_SRCIA (1 << 10) 122 #define CC_DSTIA (1 << 24) 123 #define CC_SRCBRSTLEN_SHFT 4 124 #define CC_DSTBRSTLEN_SHFT 18 125 #define CC_SRCBRSTSIZE_SHFT 1 126 #define CC_DSTBRSTSIZE_SHFT 15 127 #define CC_SRCCCTRL_SHFT 11 128 #define CC_SRCCCTRL_MASK 0x7 129 #define CC_DSTCCTRL_SHFT 25 130 #define CC_DRCCCTRL_MASK 0x7 131 #define CC_SWAP_SHFT 28 132 133 #define _LC0 0x40c 134 #define LC0(n) (_LC0 + (n)*0x20) 135 136 #define _LC1 0x410 137 #define LC1(n) (_LC1 + (n)*0x20) 138 139 #define DBGSTATUS 0xd00 140 #define DBG_BUSY (1 << 0) 141 142 #define DBGCMD 0xd04 143 #define DBGINST0 0xd08 144 #define DBGINST1 0xd0c 145 146 #define CR0 0xe00 147 #define CR1 0xe04 148 #define CR2 0xe08 149 #define CR3 0xe0c 150 #define CR4 0xe10 151 #define CRD 0xe14 152 153 #define PERIPH_ID 0xfe0 154 #define PERIPH_REV_SHIFT 20 155 #define PERIPH_REV_MASK 0xf 156 #define PERIPH_REV_R0P0 0 157 #define PERIPH_REV_R1P0 1 158 #define PERIPH_REV_R1P1 2 159 #define PCELL_ID 0xff0 160 161 #define CR0_PERIPH_REQ_SET (1 << 0) 162 #define CR0_BOOT_EN_SET (1 << 1) 163 #define CR0_BOOT_MAN_NS (1 << 2) 164 #define CR0_NUM_CHANS_SHIFT 4 165 #define CR0_NUM_CHANS_MASK 0x7 166 #define CR0_NUM_PERIPH_SHIFT 12 167 #define CR0_NUM_PERIPH_MASK 0x1f 168 #define CR0_NUM_EVENTS_SHIFT 17 169 #define CR0_NUM_EVENTS_MASK 0x1f 170 171 #define CR1_ICACHE_LEN_SHIFT 0 172 #define CR1_ICACHE_LEN_MASK 0x7 173 #define CR1_NUM_ICACHELINES_SHIFT 4 174 #define CR1_NUM_ICACHELINES_MASK 0xf 175 176 #define CRD_DATA_WIDTH_SHIFT 0 177 #define CRD_DATA_WIDTH_MASK 0x7 178 #define CRD_WR_CAP_SHIFT 4 179 #define CRD_WR_CAP_MASK 0x7 180 #define CRD_WR_Q_DEP_SHIFT 8 181 #define CRD_WR_Q_DEP_MASK 0xf 182 #define CRD_RD_CAP_SHIFT 12 183 #define CRD_RD_CAP_MASK 0x7 184 #define CRD_RD_Q_DEP_SHIFT 16 185 #define CRD_RD_Q_DEP_MASK 0xf 186 #define CRD_DATA_BUFF_SHIFT 20 187 #define CRD_DATA_BUFF_MASK 0x3ff 188 189 #define PART 0x330 190 #define DESIGNER 0x41 191 #define REVISION 0x0 192 #define INTEG_CFG 0x0 193 #define PERIPH_ID_VAL ((PART << 0) | (DESIGNER << 12)) 194 195 #define PCELL_ID_VAL 0xb105f00d 196 197 #define PL330_STATE_STOPPED (1 << 0) 198 #define PL330_STATE_EXECUTING (1 << 1) 199 #define PL330_STATE_WFE (1 << 2) 200 #define PL330_STATE_FAULTING (1 << 3) 201 #define PL330_STATE_COMPLETING (1 << 4) 202 #define PL330_STATE_WFP (1 << 5) 203 #define PL330_STATE_KILLING (1 << 6) 204 #define PL330_STATE_FAULT_COMPLETING (1 << 7) 205 #define PL330_STATE_CACHEMISS (1 << 8) 206 #define PL330_STATE_UPDTPC (1 << 9) 207 #define PL330_STATE_ATBARRIER (1 << 10) 208 #define PL330_STATE_QUEUEBUSY (1 << 11) 209 #define PL330_STATE_INVALID (1 << 15) 210 211 #define PL330_STABLE_STATES (PL330_STATE_STOPPED | PL330_STATE_EXECUTING \ 212 | PL330_STATE_WFE | PL330_STATE_FAULTING) 213 214 #define CMD_DMAADDH 0x54 215 #define CMD_DMAEND 0x00 216 #define CMD_DMAFLUSHP 0x35 217 #define CMD_DMAGO 0xa0 218 #define CMD_DMALD 0x04 219 #define CMD_DMALDP 0x25 220 #define CMD_DMALP 0x20 221 #define CMD_DMALPEND 0x28 222 #define CMD_DMAKILL 0x01 223 #define CMD_DMAMOV 0xbc 224 #define CMD_DMANOP 0x18 225 #define CMD_DMARMB 0x12 226 #define CMD_DMASEV 0x34 227 #define CMD_DMAST 0x08 228 #define CMD_DMASTP 0x29 229 #define CMD_DMASTZ 0x0c 230 #define CMD_DMAWFE 0x36 231 #define CMD_DMAWFP 0x30 232 #define CMD_DMAWMB 0x13 233 234 #define SZ_DMAADDH 3 235 #define SZ_DMAEND 1 236 #define SZ_DMAFLUSHP 2 237 #define SZ_DMALD 1 238 #define SZ_DMALDP 2 239 #define SZ_DMALP 2 240 #define SZ_DMALPEND 2 241 #define SZ_DMAKILL 1 242 #define SZ_DMAMOV 6 243 #define SZ_DMANOP 1 244 #define SZ_DMARMB 1 245 #define SZ_DMASEV 2 246 #define SZ_DMAST 1 247 #define SZ_DMASTP 2 248 #define SZ_DMASTZ 1 249 #define SZ_DMAWFE 2 250 #define SZ_DMAWFP 2 251 #define SZ_DMAWMB 1 252 #define SZ_DMAGO 6 253 254 #define BRST_LEN(ccr) ((((ccr) >> CC_SRCBRSTLEN_SHFT) & 0xf) + 1) 255 #define BRST_SIZE(ccr) (1 << (((ccr) >> CC_SRCBRSTSIZE_SHFT) & 0x7)) 256 257 #define BYTE_TO_BURST(b, ccr) ((b) / BRST_SIZE(ccr) / BRST_LEN(ccr)) 258 #define BURST_TO_BYTE(c, ccr) ((c) * BRST_SIZE(ccr) * BRST_LEN(ccr)) 259 260 /* 261 * With 256 bytes, we can do more than 2.5MB and 5MB xfers per req 262 * at 1byte/burst for P<->M and M<->M respectively. 263 * For typical scenario, at 1word/burst, 10MB and 20MB xfers per req 264 * should be enough for P<->M and M<->M respectively. 265 */ 266 #define MCODE_BUFF_PER_REQ 256 267 268 /* If the _pl330_req is available to the client */ 269 #define IS_FREE(req) (*((u8 *)((req)->mc_cpu)) == CMD_DMAEND) 270 271 /* Use this _only_ to wait on transient states */ 272 #define UNTIL(t, s) while (!(_state(t) & (s))) cpu_relax(); 273 274 #ifdef PL330_DEBUG_MCGEN 275 static unsigned cmd_line; 276 #define PL330_DBGCMD_DUMP(off, x...) do { \ 277 printk("%x:", cmd_line); \ 278 printk(x); \ 279 cmd_line += off; \ 280 } while (0) 281 #define PL330_DBGMC_START(addr) (cmd_line = addr) 282 #else 283 #define PL330_DBGCMD_DUMP(off, x...) do {} while (0) 284 #define PL330_DBGMC_START(addr) do {} while (0) 285 #endif 286 287 /* The number of default descriptors */ 288 289 #define NR_DEFAULT_DESC 16 290 291 /* Populated by the PL330 core driver for DMA API driver's info */ 292 struct pl330_config { 293 u32 periph_id; 294 u32 pcell_id; 295 #define DMAC_MODE_NS (1 << 0) 296 unsigned int mode; 297 unsigned int data_bus_width:10; /* In number of bits */ 298 unsigned int data_buf_dep:10; 299 unsigned int num_chan:4; 300 unsigned int num_peri:6; 301 u32 peri_ns; 302 unsigned int num_events:6; 303 u32 irq_ns; 304 }; 305 306 /* Handle to the DMAC provided to the PL330 core */ 307 struct pl330_info { 308 /* Owning device */ 309 struct device *dev; 310 /* Size of MicroCode buffers for each channel. */ 311 unsigned mcbufsz; 312 /* ioremap'ed address of PL330 registers. */ 313 void __iomem *base; 314 /* Client can freely use it. */ 315 void *client_data; 316 /* PL330 core data, Client must not touch it. */ 317 void *pl330_data; 318 /* Populated by the PL330 core driver during pl330_add */ 319 struct pl330_config pcfg; 320 /* 321 * If the DMAC has some reset mechanism, then the 322 * client may want to provide pointer to the method. 323 */ 324 void (*dmac_reset)(struct pl330_info *pi); 325 }; 326 327 /** 328 * Request Configuration. 329 * The PL330 core does not modify this and uses the last 330 * working configuration if the request doesn't provide any. 331 * 332 * The Client may want to provide this info only for the 333 * first request and a request with new settings. 334 */ 335 struct pl330_reqcfg { 336 /* Address Incrementing */ 337 unsigned dst_inc:1; 338 unsigned src_inc:1; 339 340 /* 341 * For now, the SRC & DST protection levels 342 * and burst size/length are assumed same. 343 */ 344 bool nonsecure; 345 bool privileged; 346 bool insnaccess; 347 unsigned brst_len:5; 348 unsigned brst_size:3; /* in power of 2 */ 349 350 enum pl330_dstcachectrl dcctl; 351 enum pl330_srccachectrl scctl; 352 enum pl330_byteswap swap; 353 struct pl330_config *pcfg; 354 }; 355 356 /* 357 * One cycle of DMAC operation. 358 * There may be more than one xfer in a request. 359 */ 360 struct pl330_xfer { 361 u32 src_addr; 362 u32 dst_addr; 363 /* Size to xfer */ 364 u32 bytes; 365 /* 366 * Pointer to next xfer in the list. 367 * The last xfer in the req must point to NULL. 368 */ 369 struct pl330_xfer *next; 370 }; 371 372 /* The xfer callbacks are made with one of these arguments. */ 373 enum pl330_op_err { 374 /* The all xfers in the request were success. */ 375 PL330_ERR_NONE, 376 /* If req aborted due to global error. */ 377 PL330_ERR_ABORT, 378 /* If req failed due to problem with Channel. */ 379 PL330_ERR_FAIL, 380 }; 381 382 /* A request defining Scatter-Gather List ending with NULL xfer. */ 383 struct pl330_req { 384 enum pl330_reqtype rqtype; 385 /* Index of peripheral for the xfer. */ 386 unsigned peri:5; 387 /* Unique token for this xfer, set by the client. */ 388 void *token; 389 /* Callback to be called after xfer. */ 390 void (*xfer_cb)(void *token, enum pl330_op_err err); 391 /* If NULL, req will be done at last set parameters. */ 392 struct pl330_reqcfg *cfg; 393 /* Pointer to first xfer in the request. */ 394 struct pl330_xfer *x; 395 /* Hook to attach to DMAC's list of reqs with due callback */ 396 struct list_head rqd; 397 }; 398 399 /* 400 * To know the status of the channel and DMAC, the client 401 * provides a pointer to this structure. The PL330 core 402 * fills it with current information. 403 */ 404 struct pl330_chanstatus { 405 /* 406 * If the DMAC engine halted due to some error, 407 * the client should remove-add DMAC. 408 */ 409 bool dmac_halted; 410 /* 411 * If channel is halted due to some error, 412 * the client should ABORT/FLUSH and START the channel. 413 */ 414 bool faulting; 415 /* Location of last load */ 416 u32 src_addr; 417 /* Location of last store */ 418 u32 dst_addr; 419 /* 420 * Pointer to the currently active req, NULL if channel is 421 * inactive, even though the requests may be present. 422 */ 423 struct pl330_req *top_req; 424 /* Pointer to req waiting second in the queue if any. */ 425 struct pl330_req *wait_req; 426 }; 427 428 enum pl330_chan_op { 429 /* Start the channel */ 430 PL330_OP_START, 431 /* Abort the active xfer */ 432 PL330_OP_ABORT, 433 /* Stop xfer and flush queue */ 434 PL330_OP_FLUSH, 435 }; 436 437 struct _xfer_spec { 438 u32 ccr; 439 struct pl330_req *r; 440 struct pl330_xfer *x; 441 }; 442 443 enum dmamov_dst { 444 SAR = 0, 445 CCR, 446 DAR, 447 }; 448 449 enum pl330_dst { 450 SRC = 0, 451 DST, 452 }; 453 454 enum pl330_cond { 455 SINGLE, 456 BURST, 457 ALWAYS, 458 }; 459 460 struct _pl330_req { 461 u32 mc_bus; 462 void *mc_cpu; 463 /* Number of bytes taken to setup MC for the req */ 464 u32 mc_len; 465 struct pl330_req *r; 466 }; 467 468 /* ToBeDone for tasklet */ 469 struct _pl330_tbd { 470 bool reset_dmac; 471 bool reset_mngr; 472 u8 reset_chan; 473 }; 474 475 /* A DMAC Thread */ 476 struct pl330_thread { 477 u8 id; 478 int ev; 479 /* If the channel is not yet acquired by any client */ 480 bool free; 481 /* Parent DMAC */ 482 struct pl330_dmac *dmac; 483 /* Only two at a time */ 484 struct _pl330_req req[2]; 485 /* Index of the last enqueued request */ 486 unsigned lstenq; 487 /* Index of the last submitted request or -1 if the DMA is stopped */ 488 int req_running; 489 }; 490 491 enum pl330_dmac_state { 492 UNINIT, 493 INIT, 494 DYING, 495 }; 496 497 /* A DMAC */ 498 struct pl330_dmac { 499 spinlock_t lock; 500 /* Holds list of reqs with due callbacks */ 501 struct list_head req_done; 502 /* Pointer to platform specific stuff */ 503 struct pl330_info *pinfo; 504 /* Maximum possible events/irqs */ 505 int events[32]; 506 /* BUS address of MicroCode buffer */ 507 u32 mcode_bus; 508 /* CPU address of MicroCode buffer */ 509 void *mcode_cpu; 510 /* List of all Channel threads */ 511 struct pl330_thread *channels; 512 /* Pointer to the MANAGER thread */ 513 struct pl330_thread *manager; 514 /* To handle bad news in interrupt */ 515 struct tasklet_struct tasks; 516 struct _pl330_tbd dmac_tbd; 517 /* State of DMAC operation */ 518 enum pl330_dmac_state state; 519 }; 520 521 enum desc_status { 522 /* In the DMAC pool */ 523 FREE, 524 /* 525 * Allocated to some channel during prep_xxx 526 * Also may be sitting on the work_list. 527 */ 528 PREP, 529 /* 530 * Sitting on the work_list and already submitted 531 * to the PL330 core. Not more than two descriptors 532 * of a channel can be BUSY at any time. 533 */ 534 BUSY, 535 /* 536 * Sitting on the channel work_list but xfer done 537 * by PL330 core 538 */ 539 DONE, 540 }; 541 542 struct dma_pl330_chan { 543 /* Schedule desc completion */ 544 struct tasklet_struct task; 545 546 /* DMA-Engine Channel */ 547 struct dma_chan chan; 548 549 /* List of to be xfered descriptors */ 550 struct list_head work_list; 551 552 /* Pointer to the DMAC that manages this channel, 553 * NULL if the channel is available to be acquired. 554 * As the parent, this DMAC also provides descriptors 555 * to the channel. 556 */ 557 struct dma_pl330_dmac *dmac; 558 559 /* To protect channel manipulation */ 560 spinlock_t lock; 561 562 /* Token of a hardware channel thread of PL330 DMAC 563 * NULL if the channel is available to be acquired. 564 */ 565 void *pl330_chid; 566 567 /* For D-to-M and M-to-D channels */ 568 int burst_sz; /* the peripheral fifo width */ 569 int burst_len; /* the number of burst */ 570 dma_addr_t fifo_addr; 571 572 /* for cyclic capability */ 573 bool cyclic; 574 }; 575 576 struct dma_pl330_dmac { 577 struct pl330_info pif; 578 579 /* DMA-Engine Device */ 580 struct dma_device ddma; 581 582 /* Pool of descriptors available for the DMAC's channels */ 583 struct list_head desc_pool; 584 /* To protect desc_pool manipulation */ 585 spinlock_t pool_lock; 586 587 /* Peripheral channels connected to this DMAC */ 588 struct dma_pl330_chan *peripherals; /* keep at end */ 589 590 struct clk *clk; 591 }; 592 593 struct dma_pl330_desc { 594 /* To attach to a queue as child */ 595 struct list_head node; 596 597 /* Descriptor for the DMA Engine API */ 598 struct dma_async_tx_descriptor txd; 599 600 /* Xfer for PL330 core */ 601 struct pl330_xfer px; 602 603 struct pl330_reqcfg rqcfg; 604 struct pl330_req req; 605 606 enum desc_status status; 607 608 /* The channel which currently holds this desc */ 609 struct dma_pl330_chan *pchan; 610 }; 611 612 static inline void _callback(struct pl330_req *r, enum pl330_op_err err) 613 { 614 if (r && r->xfer_cb) 615 r->xfer_cb(r->token, err); 616 } 617 618 static inline bool _queue_empty(struct pl330_thread *thrd) 619 { 620 return (IS_FREE(&thrd->req[0]) && IS_FREE(&thrd->req[1])) 621 ? true : false; 622 } 623 624 static inline bool _queue_full(struct pl330_thread *thrd) 625 { 626 return (IS_FREE(&thrd->req[0]) || IS_FREE(&thrd->req[1])) 627 ? false : true; 628 } 629 630 static inline bool is_manager(struct pl330_thread *thrd) 631 { 632 struct pl330_dmac *pl330 = thrd->dmac; 633 634 /* MANAGER is indexed at the end */ 635 if (thrd->id == pl330->pinfo->pcfg.num_chan) 636 return true; 637 else 638 return false; 639 } 640 641 /* If manager of the thread is in Non-Secure mode */ 642 static inline bool _manager_ns(struct pl330_thread *thrd) 643 { 644 struct pl330_dmac *pl330 = thrd->dmac; 645 646 return (pl330->pinfo->pcfg.mode & DMAC_MODE_NS) ? true : false; 647 } 648 649 static inline u32 get_id(struct pl330_info *pi, u32 off) 650 { 651 void __iomem *regs = pi->base; 652 u32 id = 0; 653 654 id |= (readb(regs + off + 0x0) << 0); 655 id |= (readb(regs + off + 0x4) << 8); 656 id |= (readb(regs + off + 0x8) << 16); 657 id |= (readb(regs + off + 0xc) << 24); 658 659 return id; 660 } 661 662 static inline u32 get_revision(u32 periph_id) 663 { 664 return (periph_id >> PERIPH_REV_SHIFT) & PERIPH_REV_MASK; 665 } 666 667 static inline u32 _emit_ADDH(unsigned dry_run, u8 buf[], 668 enum pl330_dst da, u16 val) 669 { 670 if (dry_run) 671 return SZ_DMAADDH; 672 673 buf[0] = CMD_DMAADDH; 674 buf[0] |= (da << 1); 675 *((u16 *)&buf[1]) = val; 676 677 PL330_DBGCMD_DUMP(SZ_DMAADDH, "\tDMAADDH %s %u\n", 678 da == 1 ? "DA" : "SA", val); 679 680 return SZ_DMAADDH; 681 } 682 683 static inline u32 _emit_END(unsigned dry_run, u8 buf[]) 684 { 685 if (dry_run) 686 return SZ_DMAEND; 687 688 buf[0] = CMD_DMAEND; 689 690 PL330_DBGCMD_DUMP(SZ_DMAEND, "\tDMAEND\n"); 691 692 return SZ_DMAEND; 693 } 694 695 static inline u32 _emit_FLUSHP(unsigned dry_run, u8 buf[], u8 peri) 696 { 697 if (dry_run) 698 return SZ_DMAFLUSHP; 699 700 buf[0] = CMD_DMAFLUSHP; 701 702 peri &= 0x1f; 703 peri <<= 3; 704 buf[1] = peri; 705 706 PL330_DBGCMD_DUMP(SZ_DMAFLUSHP, "\tDMAFLUSHP %u\n", peri >> 3); 707 708 return SZ_DMAFLUSHP; 709 } 710 711 static inline u32 _emit_LD(unsigned dry_run, u8 buf[], enum pl330_cond cond) 712 { 713 if (dry_run) 714 return SZ_DMALD; 715 716 buf[0] = CMD_DMALD; 717 718 if (cond == SINGLE) 719 buf[0] |= (0 << 1) | (1 << 0); 720 else if (cond == BURST) 721 buf[0] |= (1 << 1) | (1 << 0); 722 723 PL330_DBGCMD_DUMP(SZ_DMALD, "\tDMALD%c\n", 724 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); 725 726 return SZ_DMALD; 727 } 728 729 static inline u32 _emit_LDP(unsigned dry_run, u8 buf[], 730 enum pl330_cond cond, u8 peri) 731 { 732 if (dry_run) 733 return SZ_DMALDP; 734 735 buf[0] = CMD_DMALDP; 736 737 if (cond == BURST) 738 buf[0] |= (1 << 1); 739 740 peri &= 0x1f; 741 peri <<= 3; 742 buf[1] = peri; 743 744 PL330_DBGCMD_DUMP(SZ_DMALDP, "\tDMALDP%c %u\n", 745 cond == SINGLE ? 'S' : 'B', peri >> 3); 746 747 return SZ_DMALDP; 748 } 749 750 static inline u32 _emit_LP(unsigned dry_run, u8 buf[], 751 unsigned loop, u8 cnt) 752 { 753 if (dry_run) 754 return SZ_DMALP; 755 756 buf[0] = CMD_DMALP; 757 758 if (loop) 759 buf[0] |= (1 << 1); 760 761 cnt--; /* DMAC increments by 1 internally */ 762 buf[1] = cnt; 763 764 PL330_DBGCMD_DUMP(SZ_DMALP, "\tDMALP_%c %u\n", loop ? '1' : '0', cnt); 765 766 return SZ_DMALP; 767 } 768 769 struct _arg_LPEND { 770 enum pl330_cond cond; 771 bool forever; 772 unsigned loop; 773 u8 bjump; 774 }; 775 776 static inline u32 _emit_LPEND(unsigned dry_run, u8 buf[], 777 const struct _arg_LPEND *arg) 778 { 779 enum pl330_cond cond = arg->cond; 780 bool forever = arg->forever; 781 unsigned loop = arg->loop; 782 u8 bjump = arg->bjump; 783 784 if (dry_run) 785 return SZ_DMALPEND; 786 787 buf[0] = CMD_DMALPEND; 788 789 if (loop) 790 buf[0] |= (1 << 2); 791 792 if (!forever) 793 buf[0] |= (1 << 4); 794 795 if (cond == SINGLE) 796 buf[0] |= (0 << 1) | (1 << 0); 797 else if (cond == BURST) 798 buf[0] |= (1 << 1) | (1 << 0); 799 800 buf[1] = bjump; 801 802 PL330_DBGCMD_DUMP(SZ_DMALPEND, "\tDMALP%s%c_%c bjmpto_%x\n", 803 forever ? "FE" : "END", 804 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A'), 805 loop ? '1' : '0', 806 bjump); 807 808 return SZ_DMALPEND; 809 } 810 811 static inline u32 _emit_KILL(unsigned dry_run, u8 buf[]) 812 { 813 if (dry_run) 814 return SZ_DMAKILL; 815 816 buf[0] = CMD_DMAKILL; 817 818 return SZ_DMAKILL; 819 } 820 821 static inline u32 _emit_MOV(unsigned dry_run, u8 buf[], 822 enum dmamov_dst dst, u32 val) 823 { 824 if (dry_run) 825 return SZ_DMAMOV; 826 827 buf[0] = CMD_DMAMOV; 828 buf[1] = dst; 829 *((u32 *)&buf[2]) = val; 830 831 PL330_DBGCMD_DUMP(SZ_DMAMOV, "\tDMAMOV %s 0x%x\n", 832 dst == SAR ? "SAR" : (dst == DAR ? "DAR" : "CCR"), val); 833 834 return SZ_DMAMOV; 835 } 836 837 static inline u32 _emit_NOP(unsigned dry_run, u8 buf[]) 838 { 839 if (dry_run) 840 return SZ_DMANOP; 841 842 buf[0] = CMD_DMANOP; 843 844 PL330_DBGCMD_DUMP(SZ_DMANOP, "\tDMANOP\n"); 845 846 return SZ_DMANOP; 847 } 848 849 static inline u32 _emit_RMB(unsigned dry_run, u8 buf[]) 850 { 851 if (dry_run) 852 return SZ_DMARMB; 853 854 buf[0] = CMD_DMARMB; 855 856 PL330_DBGCMD_DUMP(SZ_DMARMB, "\tDMARMB\n"); 857 858 return SZ_DMARMB; 859 } 860 861 static inline u32 _emit_SEV(unsigned dry_run, u8 buf[], u8 ev) 862 { 863 if (dry_run) 864 return SZ_DMASEV; 865 866 buf[0] = CMD_DMASEV; 867 868 ev &= 0x1f; 869 ev <<= 3; 870 buf[1] = ev; 871 872 PL330_DBGCMD_DUMP(SZ_DMASEV, "\tDMASEV %u\n", ev >> 3); 873 874 return SZ_DMASEV; 875 } 876 877 static inline u32 _emit_ST(unsigned dry_run, u8 buf[], enum pl330_cond cond) 878 { 879 if (dry_run) 880 return SZ_DMAST; 881 882 buf[0] = CMD_DMAST; 883 884 if (cond == SINGLE) 885 buf[0] |= (0 << 1) | (1 << 0); 886 else if (cond == BURST) 887 buf[0] |= (1 << 1) | (1 << 0); 888 889 PL330_DBGCMD_DUMP(SZ_DMAST, "\tDMAST%c\n", 890 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'A')); 891 892 return SZ_DMAST; 893 } 894 895 static inline u32 _emit_STP(unsigned dry_run, u8 buf[], 896 enum pl330_cond cond, u8 peri) 897 { 898 if (dry_run) 899 return SZ_DMASTP; 900 901 buf[0] = CMD_DMASTP; 902 903 if (cond == BURST) 904 buf[0] |= (1 << 1); 905 906 peri &= 0x1f; 907 peri <<= 3; 908 buf[1] = peri; 909 910 PL330_DBGCMD_DUMP(SZ_DMASTP, "\tDMASTP%c %u\n", 911 cond == SINGLE ? 'S' : 'B', peri >> 3); 912 913 return SZ_DMASTP; 914 } 915 916 static inline u32 _emit_STZ(unsigned dry_run, u8 buf[]) 917 { 918 if (dry_run) 919 return SZ_DMASTZ; 920 921 buf[0] = CMD_DMASTZ; 922 923 PL330_DBGCMD_DUMP(SZ_DMASTZ, "\tDMASTZ\n"); 924 925 return SZ_DMASTZ; 926 } 927 928 static inline u32 _emit_WFE(unsigned dry_run, u8 buf[], u8 ev, 929 unsigned invalidate) 930 { 931 if (dry_run) 932 return SZ_DMAWFE; 933 934 buf[0] = CMD_DMAWFE; 935 936 ev &= 0x1f; 937 ev <<= 3; 938 buf[1] = ev; 939 940 if (invalidate) 941 buf[1] |= (1 << 1); 942 943 PL330_DBGCMD_DUMP(SZ_DMAWFE, "\tDMAWFE %u%s\n", 944 ev >> 3, invalidate ? ", I" : ""); 945 946 return SZ_DMAWFE; 947 } 948 949 static inline u32 _emit_WFP(unsigned dry_run, u8 buf[], 950 enum pl330_cond cond, u8 peri) 951 { 952 if (dry_run) 953 return SZ_DMAWFP; 954 955 buf[0] = CMD_DMAWFP; 956 957 if (cond == SINGLE) 958 buf[0] |= (0 << 1) | (0 << 0); 959 else if (cond == BURST) 960 buf[0] |= (1 << 1) | (0 << 0); 961 else 962 buf[0] |= (0 << 1) | (1 << 0); 963 964 peri &= 0x1f; 965 peri <<= 3; 966 buf[1] = peri; 967 968 PL330_DBGCMD_DUMP(SZ_DMAWFP, "\tDMAWFP%c %u\n", 969 cond == SINGLE ? 'S' : (cond == BURST ? 'B' : 'P'), peri >> 3); 970 971 return SZ_DMAWFP; 972 } 973 974 static inline u32 _emit_WMB(unsigned dry_run, u8 buf[]) 975 { 976 if (dry_run) 977 return SZ_DMAWMB; 978 979 buf[0] = CMD_DMAWMB; 980 981 PL330_DBGCMD_DUMP(SZ_DMAWMB, "\tDMAWMB\n"); 982 983 return SZ_DMAWMB; 984 } 985 986 struct _arg_GO { 987 u8 chan; 988 u32 addr; 989 unsigned ns; 990 }; 991 992 static inline u32 _emit_GO(unsigned dry_run, u8 buf[], 993 const struct _arg_GO *arg) 994 { 995 u8 chan = arg->chan; 996 u32 addr = arg->addr; 997 unsigned ns = arg->ns; 998 999 if (dry_run) 1000 return SZ_DMAGO; 1001 1002 buf[0] = CMD_DMAGO; 1003 buf[0] |= (ns << 1); 1004 1005 buf[1] = chan & 0x7; 1006 1007 *((u32 *)&buf[2]) = addr; 1008 1009 return SZ_DMAGO; 1010 } 1011 1012 #define msecs_to_loops(t) (loops_per_jiffy / 1000 * HZ * t) 1013 1014 /* Returns Time-Out */ 1015 static bool _until_dmac_idle(struct pl330_thread *thrd) 1016 { 1017 void __iomem *regs = thrd->dmac->pinfo->base; 1018 unsigned long loops = msecs_to_loops(5); 1019 1020 do { 1021 /* Until Manager is Idle */ 1022 if (!(readl(regs + DBGSTATUS) & DBG_BUSY)) 1023 break; 1024 1025 cpu_relax(); 1026 } while (--loops); 1027 1028 if (!loops) 1029 return true; 1030 1031 return false; 1032 } 1033 1034 static inline void _execute_DBGINSN(struct pl330_thread *thrd, 1035 u8 insn[], bool as_manager) 1036 { 1037 void __iomem *regs = thrd->dmac->pinfo->base; 1038 u32 val; 1039 1040 val = (insn[0] << 16) | (insn[1] << 24); 1041 if (!as_manager) { 1042 val |= (1 << 0); 1043 val |= (thrd->id << 8); /* Channel Number */ 1044 } 1045 writel(val, regs + DBGINST0); 1046 1047 val = *((u32 *)&insn[2]); 1048 writel(val, regs + DBGINST1); 1049 1050 /* If timed out due to halted state-machine */ 1051 if (_until_dmac_idle(thrd)) { 1052 dev_err(thrd->dmac->pinfo->dev, "DMAC halted!\n"); 1053 return; 1054 } 1055 1056 /* Get going */ 1057 writel(0, regs + DBGCMD); 1058 } 1059 1060 /* 1061 * Mark a _pl330_req as free. 1062 * We do it by writing DMAEND as the first instruction 1063 * because no valid request is going to have DMAEND as 1064 * its first instruction to execute. 1065 */ 1066 static void mark_free(struct pl330_thread *thrd, int idx) 1067 { 1068 struct _pl330_req *req = &thrd->req[idx]; 1069 1070 _emit_END(0, req->mc_cpu); 1071 req->mc_len = 0; 1072 1073 thrd->req_running = -1; 1074 } 1075 1076 static inline u32 _state(struct pl330_thread *thrd) 1077 { 1078 void __iomem *regs = thrd->dmac->pinfo->base; 1079 u32 val; 1080 1081 if (is_manager(thrd)) 1082 val = readl(regs + DS) & 0xf; 1083 else 1084 val = readl(regs + CS(thrd->id)) & 0xf; 1085 1086 switch (val) { 1087 case DS_ST_STOP: 1088 return PL330_STATE_STOPPED; 1089 case DS_ST_EXEC: 1090 return PL330_STATE_EXECUTING; 1091 case DS_ST_CMISS: 1092 return PL330_STATE_CACHEMISS; 1093 case DS_ST_UPDTPC: 1094 return PL330_STATE_UPDTPC; 1095 case DS_ST_WFE: 1096 return PL330_STATE_WFE; 1097 case DS_ST_FAULT: 1098 return PL330_STATE_FAULTING; 1099 case DS_ST_ATBRR: 1100 if (is_manager(thrd)) 1101 return PL330_STATE_INVALID; 1102 else 1103 return PL330_STATE_ATBARRIER; 1104 case DS_ST_QBUSY: 1105 if (is_manager(thrd)) 1106 return PL330_STATE_INVALID; 1107 else 1108 return PL330_STATE_QUEUEBUSY; 1109 case DS_ST_WFP: 1110 if (is_manager(thrd)) 1111 return PL330_STATE_INVALID; 1112 else 1113 return PL330_STATE_WFP; 1114 case DS_ST_KILL: 1115 if (is_manager(thrd)) 1116 return PL330_STATE_INVALID; 1117 else 1118 return PL330_STATE_KILLING; 1119 case DS_ST_CMPLT: 1120 if (is_manager(thrd)) 1121 return PL330_STATE_INVALID; 1122 else 1123 return PL330_STATE_COMPLETING; 1124 case DS_ST_FLTCMP: 1125 if (is_manager(thrd)) 1126 return PL330_STATE_INVALID; 1127 else 1128 return PL330_STATE_FAULT_COMPLETING; 1129 default: 1130 return PL330_STATE_INVALID; 1131 } 1132 } 1133 1134 static void _stop(struct pl330_thread *thrd) 1135 { 1136 void __iomem *regs = thrd->dmac->pinfo->base; 1137 u8 insn[6] = {0, 0, 0, 0, 0, 0}; 1138 1139 if (_state(thrd) == PL330_STATE_FAULT_COMPLETING) 1140 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING); 1141 1142 /* Return if nothing needs to be done */ 1143 if (_state(thrd) == PL330_STATE_COMPLETING 1144 || _state(thrd) == PL330_STATE_KILLING 1145 || _state(thrd) == PL330_STATE_STOPPED) 1146 return; 1147 1148 _emit_KILL(0, insn); 1149 1150 /* Stop generating interrupts for SEV */ 1151 writel(readl(regs + INTEN) & ~(1 << thrd->ev), regs + INTEN); 1152 1153 _execute_DBGINSN(thrd, insn, is_manager(thrd)); 1154 } 1155 1156 /* Start doing req 'idx' of thread 'thrd' */ 1157 static bool _trigger(struct pl330_thread *thrd) 1158 { 1159 void __iomem *regs = thrd->dmac->pinfo->base; 1160 struct _pl330_req *req; 1161 struct pl330_req *r; 1162 struct _arg_GO go; 1163 unsigned ns; 1164 u8 insn[6] = {0, 0, 0, 0, 0, 0}; 1165 int idx; 1166 1167 /* Return if already ACTIVE */ 1168 if (_state(thrd) != PL330_STATE_STOPPED) 1169 return true; 1170 1171 idx = 1 - thrd->lstenq; 1172 if (!IS_FREE(&thrd->req[idx])) 1173 req = &thrd->req[idx]; 1174 else { 1175 idx = thrd->lstenq; 1176 if (!IS_FREE(&thrd->req[idx])) 1177 req = &thrd->req[idx]; 1178 else 1179 req = NULL; 1180 } 1181 1182 /* Return if no request */ 1183 if (!req || !req->r) 1184 return true; 1185 1186 r = req->r; 1187 1188 if (r->cfg) 1189 ns = r->cfg->nonsecure ? 1 : 0; 1190 else if (readl(regs + CS(thrd->id)) & CS_CNS) 1191 ns = 1; 1192 else 1193 ns = 0; 1194 1195 /* See 'Abort Sources' point-4 at Page 2-25 */ 1196 if (_manager_ns(thrd) && !ns) 1197 dev_info(thrd->dmac->pinfo->dev, "%s:%d Recipe for ABORT!\n", 1198 __func__, __LINE__); 1199 1200 go.chan = thrd->id; 1201 go.addr = req->mc_bus; 1202 go.ns = ns; 1203 _emit_GO(0, insn, &go); 1204 1205 /* Set to generate interrupts for SEV */ 1206 writel(readl(regs + INTEN) | (1 << thrd->ev), regs + INTEN); 1207 1208 /* Only manager can execute GO */ 1209 _execute_DBGINSN(thrd, insn, true); 1210 1211 thrd->req_running = idx; 1212 1213 return true; 1214 } 1215 1216 static bool _start(struct pl330_thread *thrd) 1217 { 1218 switch (_state(thrd)) { 1219 case PL330_STATE_FAULT_COMPLETING: 1220 UNTIL(thrd, PL330_STATE_FAULTING | PL330_STATE_KILLING); 1221 1222 if (_state(thrd) == PL330_STATE_KILLING) 1223 UNTIL(thrd, PL330_STATE_STOPPED) 1224 1225 case PL330_STATE_FAULTING: 1226 _stop(thrd); 1227 1228 case PL330_STATE_KILLING: 1229 case PL330_STATE_COMPLETING: 1230 UNTIL(thrd, PL330_STATE_STOPPED) 1231 1232 case PL330_STATE_STOPPED: 1233 return _trigger(thrd); 1234 1235 case PL330_STATE_WFP: 1236 case PL330_STATE_QUEUEBUSY: 1237 case PL330_STATE_ATBARRIER: 1238 case PL330_STATE_UPDTPC: 1239 case PL330_STATE_CACHEMISS: 1240 case PL330_STATE_EXECUTING: 1241 return true; 1242 1243 case PL330_STATE_WFE: /* For RESUME, nothing yet */ 1244 default: 1245 return false; 1246 } 1247 } 1248 1249 static inline int _ldst_memtomem(unsigned dry_run, u8 buf[], 1250 const struct _xfer_spec *pxs, int cyc) 1251 { 1252 int off = 0; 1253 struct pl330_config *pcfg = pxs->r->cfg->pcfg; 1254 1255 /* check lock-up free version */ 1256 if (get_revision(pcfg->periph_id) >= PERIPH_REV_R1P0) { 1257 while (cyc--) { 1258 off += _emit_LD(dry_run, &buf[off], ALWAYS); 1259 off += _emit_ST(dry_run, &buf[off], ALWAYS); 1260 } 1261 } else { 1262 while (cyc--) { 1263 off += _emit_LD(dry_run, &buf[off], ALWAYS); 1264 off += _emit_RMB(dry_run, &buf[off]); 1265 off += _emit_ST(dry_run, &buf[off], ALWAYS); 1266 off += _emit_WMB(dry_run, &buf[off]); 1267 } 1268 } 1269 1270 return off; 1271 } 1272 1273 static inline int _ldst_devtomem(unsigned dry_run, u8 buf[], 1274 const struct _xfer_spec *pxs, int cyc) 1275 { 1276 int off = 0; 1277 1278 while (cyc--) { 1279 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri); 1280 off += _emit_LDP(dry_run, &buf[off], SINGLE, pxs->r->peri); 1281 off += _emit_ST(dry_run, &buf[off], ALWAYS); 1282 off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri); 1283 } 1284 1285 return off; 1286 } 1287 1288 static inline int _ldst_memtodev(unsigned dry_run, u8 buf[], 1289 const struct _xfer_spec *pxs, int cyc) 1290 { 1291 int off = 0; 1292 1293 while (cyc--) { 1294 off += _emit_WFP(dry_run, &buf[off], SINGLE, pxs->r->peri); 1295 off += _emit_LD(dry_run, &buf[off], ALWAYS); 1296 off += _emit_STP(dry_run, &buf[off], SINGLE, pxs->r->peri); 1297 off += _emit_FLUSHP(dry_run, &buf[off], pxs->r->peri); 1298 } 1299 1300 return off; 1301 } 1302 1303 static int _bursts(unsigned dry_run, u8 buf[], 1304 const struct _xfer_spec *pxs, int cyc) 1305 { 1306 int off = 0; 1307 1308 switch (pxs->r->rqtype) { 1309 case MEMTODEV: 1310 off += _ldst_memtodev(dry_run, &buf[off], pxs, cyc); 1311 break; 1312 case DEVTOMEM: 1313 off += _ldst_devtomem(dry_run, &buf[off], pxs, cyc); 1314 break; 1315 case MEMTOMEM: 1316 off += _ldst_memtomem(dry_run, &buf[off], pxs, cyc); 1317 break; 1318 default: 1319 off += 0x40000000; /* Scare off the Client */ 1320 break; 1321 } 1322 1323 return off; 1324 } 1325 1326 /* Returns bytes consumed and updates bursts */ 1327 static inline int _loop(unsigned dry_run, u8 buf[], 1328 unsigned long *bursts, const struct _xfer_spec *pxs) 1329 { 1330 int cyc, cycmax, szlp, szlpend, szbrst, off; 1331 unsigned lcnt0, lcnt1, ljmp0, ljmp1; 1332 struct _arg_LPEND lpend; 1333 1334 /* Max iterations possible in DMALP is 256 */ 1335 if (*bursts >= 256*256) { 1336 lcnt1 = 256; 1337 lcnt0 = 256; 1338 cyc = *bursts / lcnt1 / lcnt0; 1339 } else if (*bursts > 256) { 1340 lcnt1 = 256; 1341 lcnt0 = *bursts / lcnt1; 1342 cyc = 1; 1343 } else { 1344 lcnt1 = *bursts; 1345 lcnt0 = 0; 1346 cyc = 1; 1347 } 1348 1349 szlp = _emit_LP(1, buf, 0, 0); 1350 szbrst = _bursts(1, buf, pxs, 1); 1351 1352 lpend.cond = ALWAYS; 1353 lpend.forever = false; 1354 lpend.loop = 0; 1355 lpend.bjump = 0; 1356 szlpend = _emit_LPEND(1, buf, &lpend); 1357 1358 if (lcnt0) { 1359 szlp *= 2; 1360 szlpend *= 2; 1361 } 1362 1363 /* 1364 * Max bursts that we can unroll due to limit on the 1365 * size of backward jump that can be encoded in DMALPEND 1366 * which is 8-bits and hence 255 1367 */ 1368 cycmax = (255 - (szlp + szlpend)) / szbrst; 1369 1370 cyc = (cycmax < cyc) ? cycmax : cyc; 1371 1372 off = 0; 1373 1374 if (lcnt0) { 1375 off += _emit_LP(dry_run, &buf[off], 0, lcnt0); 1376 ljmp0 = off; 1377 } 1378 1379 off += _emit_LP(dry_run, &buf[off], 1, lcnt1); 1380 ljmp1 = off; 1381 1382 off += _bursts(dry_run, &buf[off], pxs, cyc); 1383 1384 lpend.cond = ALWAYS; 1385 lpend.forever = false; 1386 lpend.loop = 1; 1387 lpend.bjump = off - ljmp1; 1388 off += _emit_LPEND(dry_run, &buf[off], &lpend); 1389 1390 if (lcnt0) { 1391 lpend.cond = ALWAYS; 1392 lpend.forever = false; 1393 lpend.loop = 0; 1394 lpend.bjump = off - ljmp0; 1395 off += _emit_LPEND(dry_run, &buf[off], &lpend); 1396 } 1397 1398 *bursts = lcnt1 * cyc; 1399 if (lcnt0) 1400 *bursts *= lcnt0; 1401 1402 return off; 1403 } 1404 1405 static inline int _setup_loops(unsigned dry_run, u8 buf[], 1406 const struct _xfer_spec *pxs) 1407 { 1408 struct pl330_xfer *x = pxs->x; 1409 u32 ccr = pxs->ccr; 1410 unsigned long c, bursts = BYTE_TO_BURST(x->bytes, ccr); 1411 int off = 0; 1412 1413 while (bursts) { 1414 c = bursts; 1415 off += _loop(dry_run, &buf[off], &c, pxs); 1416 bursts -= c; 1417 } 1418 1419 return off; 1420 } 1421 1422 static inline int _setup_xfer(unsigned dry_run, u8 buf[], 1423 const struct _xfer_spec *pxs) 1424 { 1425 struct pl330_xfer *x = pxs->x; 1426 int off = 0; 1427 1428 /* DMAMOV SAR, x->src_addr */ 1429 off += _emit_MOV(dry_run, &buf[off], SAR, x->src_addr); 1430 /* DMAMOV DAR, x->dst_addr */ 1431 off += _emit_MOV(dry_run, &buf[off], DAR, x->dst_addr); 1432 1433 /* Setup Loop(s) */ 1434 off += _setup_loops(dry_run, &buf[off], pxs); 1435 1436 return off; 1437 } 1438 1439 /* 1440 * A req is a sequence of one or more xfer units. 1441 * Returns the number of bytes taken to setup the MC for the req. 1442 */ 1443 static int _setup_req(unsigned dry_run, struct pl330_thread *thrd, 1444 unsigned index, struct _xfer_spec *pxs) 1445 { 1446 struct _pl330_req *req = &thrd->req[index]; 1447 struct pl330_xfer *x; 1448 u8 *buf = req->mc_cpu; 1449 int off = 0; 1450 1451 PL330_DBGMC_START(req->mc_bus); 1452 1453 /* DMAMOV CCR, ccr */ 1454 off += _emit_MOV(dry_run, &buf[off], CCR, pxs->ccr); 1455 1456 x = pxs->r->x; 1457 do { 1458 /* Error if xfer length is not aligned at burst size */ 1459 if (x->bytes % (BRST_SIZE(pxs->ccr) * BRST_LEN(pxs->ccr))) 1460 return -EINVAL; 1461 1462 pxs->x = x; 1463 off += _setup_xfer(dry_run, &buf[off], pxs); 1464 1465 x = x->next; 1466 } while (x); 1467 1468 /* DMASEV peripheral/event */ 1469 off += _emit_SEV(dry_run, &buf[off], thrd->ev); 1470 /* DMAEND */ 1471 off += _emit_END(dry_run, &buf[off]); 1472 1473 return off; 1474 } 1475 1476 static inline u32 _prepare_ccr(const struct pl330_reqcfg *rqc) 1477 { 1478 u32 ccr = 0; 1479 1480 if (rqc->src_inc) 1481 ccr |= CC_SRCINC; 1482 1483 if (rqc->dst_inc) 1484 ccr |= CC_DSTINC; 1485 1486 /* We set same protection levels for Src and DST for now */ 1487 if (rqc->privileged) 1488 ccr |= CC_SRCPRI | CC_DSTPRI; 1489 if (rqc->nonsecure) 1490 ccr |= CC_SRCNS | CC_DSTNS; 1491 if (rqc->insnaccess) 1492 ccr |= CC_SRCIA | CC_DSTIA; 1493 1494 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_SRCBRSTLEN_SHFT); 1495 ccr |= (((rqc->brst_len - 1) & 0xf) << CC_DSTBRSTLEN_SHFT); 1496 1497 ccr |= (rqc->brst_size << CC_SRCBRSTSIZE_SHFT); 1498 ccr |= (rqc->brst_size << CC_DSTBRSTSIZE_SHFT); 1499 1500 ccr |= (rqc->scctl << CC_SRCCCTRL_SHFT); 1501 ccr |= (rqc->dcctl << CC_DSTCCTRL_SHFT); 1502 1503 ccr |= (rqc->swap << CC_SWAP_SHFT); 1504 1505 return ccr; 1506 } 1507 1508 static inline bool _is_valid(u32 ccr) 1509 { 1510 enum pl330_dstcachectrl dcctl; 1511 enum pl330_srccachectrl scctl; 1512 1513 dcctl = (ccr >> CC_DSTCCTRL_SHFT) & CC_DRCCCTRL_MASK; 1514 scctl = (ccr >> CC_SRCCCTRL_SHFT) & CC_SRCCCTRL_MASK; 1515 1516 if (dcctl == DINVALID1 || dcctl == DINVALID2 1517 || scctl == SINVALID1 || scctl == SINVALID2) 1518 return false; 1519 else 1520 return true; 1521 } 1522 1523 /* 1524 * Submit a list of xfers after which the client wants notification. 1525 * Client is not notified after each xfer unit, just once after all 1526 * xfer units are done or some error occurs. 1527 */ 1528 static int pl330_submit_req(void *ch_id, struct pl330_req *r) 1529 { 1530 struct pl330_thread *thrd = ch_id; 1531 struct pl330_dmac *pl330; 1532 struct pl330_info *pi; 1533 struct _xfer_spec xs; 1534 unsigned long flags; 1535 void __iomem *regs; 1536 unsigned idx; 1537 u32 ccr; 1538 int ret = 0; 1539 1540 /* No Req or Unacquired Channel or DMAC */ 1541 if (!r || !thrd || thrd->free) 1542 return -EINVAL; 1543 1544 pl330 = thrd->dmac; 1545 pi = pl330->pinfo; 1546 regs = pi->base; 1547 1548 if (pl330->state == DYING 1549 || pl330->dmac_tbd.reset_chan & (1 << thrd->id)) { 1550 dev_info(thrd->dmac->pinfo->dev, "%s:%d\n", 1551 __func__, __LINE__); 1552 return -EAGAIN; 1553 } 1554 1555 /* If request for non-existing peripheral */ 1556 if (r->rqtype != MEMTOMEM && r->peri >= pi->pcfg.num_peri) { 1557 dev_info(thrd->dmac->pinfo->dev, 1558 "%s:%d Invalid peripheral(%u)!\n", 1559 __func__, __LINE__, r->peri); 1560 return -EINVAL; 1561 } 1562 1563 spin_lock_irqsave(&pl330->lock, flags); 1564 1565 if (_queue_full(thrd)) { 1566 ret = -EAGAIN; 1567 goto xfer_exit; 1568 } 1569 1570 1571 /* Use last settings, if not provided */ 1572 if (r->cfg) { 1573 /* Prefer Secure Channel */ 1574 if (!_manager_ns(thrd)) 1575 r->cfg->nonsecure = 0; 1576 else 1577 r->cfg->nonsecure = 1; 1578 1579 ccr = _prepare_ccr(r->cfg); 1580 } else { 1581 ccr = readl(regs + CC(thrd->id)); 1582 } 1583 1584 /* If this req doesn't have valid xfer settings */ 1585 if (!_is_valid(ccr)) { 1586 ret = -EINVAL; 1587 dev_info(thrd->dmac->pinfo->dev, "%s:%d Invalid CCR(%x)!\n", 1588 __func__, __LINE__, ccr); 1589 goto xfer_exit; 1590 } 1591 1592 idx = IS_FREE(&thrd->req[0]) ? 0 : 1; 1593 1594 xs.ccr = ccr; 1595 xs.r = r; 1596 1597 /* First dry run to check if req is acceptable */ 1598 ret = _setup_req(1, thrd, idx, &xs); 1599 if (ret < 0) 1600 goto xfer_exit; 1601 1602 if (ret > pi->mcbufsz / 2) { 1603 dev_info(thrd->dmac->pinfo->dev, 1604 "%s:%d Trying increasing mcbufsz\n", 1605 __func__, __LINE__); 1606 ret = -ENOMEM; 1607 goto xfer_exit; 1608 } 1609 1610 /* Hook the request */ 1611 thrd->lstenq = idx; 1612 thrd->req[idx].mc_len = _setup_req(0, thrd, idx, &xs); 1613 thrd->req[idx].r = r; 1614 1615 ret = 0; 1616 1617 xfer_exit: 1618 spin_unlock_irqrestore(&pl330->lock, flags); 1619 1620 return ret; 1621 } 1622 1623 static void pl330_dotask(unsigned long data) 1624 { 1625 struct pl330_dmac *pl330 = (struct pl330_dmac *) data; 1626 struct pl330_info *pi = pl330->pinfo; 1627 unsigned long flags; 1628 int i; 1629 1630 spin_lock_irqsave(&pl330->lock, flags); 1631 1632 /* The DMAC itself gone nuts */ 1633 if (pl330->dmac_tbd.reset_dmac) { 1634 pl330->state = DYING; 1635 /* Reset the manager too */ 1636 pl330->dmac_tbd.reset_mngr = true; 1637 /* Clear the reset flag */ 1638 pl330->dmac_tbd.reset_dmac = false; 1639 } 1640 1641 if (pl330->dmac_tbd.reset_mngr) { 1642 _stop(pl330->manager); 1643 /* Reset all channels */ 1644 pl330->dmac_tbd.reset_chan = (1 << pi->pcfg.num_chan) - 1; 1645 /* Clear the reset flag */ 1646 pl330->dmac_tbd.reset_mngr = false; 1647 } 1648 1649 for (i = 0; i < pi->pcfg.num_chan; i++) { 1650 1651 if (pl330->dmac_tbd.reset_chan & (1 << i)) { 1652 struct pl330_thread *thrd = &pl330->channels[i]; 1653 void __iomem *regs = pi->base; 1654 enum pl330_op_err err; 1655 1656 _stop(thrd); 1657 1658 if (readl(regs + FSC) & (1 << thrd->id)) 1659 err = PL330_ERR_FAIL; 1660 else 1661 err = PL330_ERR_ABORT; 1662 1663 spin_unlock_irqrestore(&pl330->lock, flags); 1664 1665 _callback(thrd->req[1 - thrd->lstenq].r, err); 1666 _callback(thrd->req[thrd->lstenq].r, err); 1667 1668 spin_lock_irqsave(&pl330->lock, flags); 1669 1670 thrd->req[0].r = NULL; 1671 thrd->req[1].r = NULL; 1672 mark_free(thrd, 0); 1673 mark_free(thrd, 1); 1674 1675 /* Clear the reset flag */ 1676 pl330->dmac_tbd.reset_chan &= ~(1 << i); 1677 } 1678 } 1679 1680 spin_unlock_irqrestore(&pl330->lock, flags); 1681 1682 return; 1683 } 1684 1685 /* Returns 1 if state was updated, 0 otherwise */ 1686 static int pl330_update(const struct pl330_info *pi) 1687 { 1688 struct pl330_req *rqdone, *tmp; 1689 struct pl330_dmac *pl330; 1690 unsigned long flags; 1691 void __iomem *regs; 1692 u32 val; 1693 int id, ev, ret = 0; 1694 1695 if (!pi || !pi->pl330_data) 1696 return 0; 1697 1698 regs = pi->base; 1699 pl330 = pi->pl330_data; 1700 1701 spin_lock_irqsave(&pl330->lock, flags); 1702 1703 val = readl(regs + FSM) & 0x1; 1704 if (val) 1705 pl330->dmac_tbd.reset_mngr = true; 1706 else 1707 pl330->dmac_tbd.reset_mngr = false; 1708 1709 val = readl(regs + FSC) & ((1 << pi->pcfg.num_chan) - 1); 1710 pl330->dmac_tbd.reset_chan |= val; 1711 if (val) { 1712 int i = 0; 1713 while (i < pi->pcfg.num_chan) { 1714 if (val & (1 << i)) { 1715 dev_info(pi->dev, 1716 "Reset Channel-%d\t CS-%x FTC-%x\n", 1717 i, readl(regs + CS(i)), 1718 readl(regs + FTC(i))); 1719 _stop(&pl330->channels[i]); 1720 } 1721 i++; 1722 } 1723 } 1724 1725 /* Check which event happened i.e, thread notified */ 1726 val = readl(regs + ES); 1727 if (pi->pcfg.num_events < 32 1728 && val & ~((1 << pi->pcfg.num_events) - 1)) { 1729 pl330->dmac_tbd.reset_dmac = true; 1730 dev_err(pi->dev, "%s:%d Unexpected!\n", __func__, __LINE__); 1731 ret = 1; 1732 goto updt_exit; 1733 } 1734 1735 for (ev = 0; ev < pi->pcfg.num_events; ev++) { 1736 if (val & (1 << ev)) { /* Event occurred */ 1737 struct pl330_thread *thrd; 1738 u32 inten = readl(regs + INTEN); 1739 int active; 1740 1741 /* Clear the event */ 1742 if (inten & (1 << ev)) 1743 writel(1 << ev, regs + INTCLR); 1744 1745 ret = 1; 1746 1747 id = pl330->events[ev]; 1748 1749 thrd = &pl330->channels[id]; 1750 1751 active = thrd->req_running; 1752 if (active == -1) /* Aborted */ 1753 continue; 1754 1755 /* Detach the req */ 1756 rqdone = thrd->req[active].r; 1757 thrd->req[active].r = NULL; 1758 1759 mark_free(thrd, active); 1760 1761 /* Get going again ASAP */ 1762 _start(thrd); 1763 1764 /* For now, just make a list of callbacks to be done */ 1765 list_add_tail(&rqdone->rqd, &pl330->req_done); 1766 } 1767 } 1768 1769 /* Now that we are in no hurry, do the callbacks */ 1770 list_for_each_entry_safe(rqdone, tmp, &pl330->req_done, rqd) { 1771 list_del(&rqdone->rqd); 1772 1773 spin_unlock_irqrestore(&pl330->lock, flags); 1774 _callback(rqdone, PL330_ERR_NONE); 1775 spin_lock_irqsave(&pl330->lock, flags); 1776 } 1777 1778 updt_exit: 1779 spin_unlock_irqrestore(&pl330->lock, flags); 1780 1781 if (pl330->dmac_tbd.reset_dmac 1782 || pl330->dmac_tbd.reset_mngr 1783 || pl330->dmac_tbd.reset_chan) { 1784 ret = 1; 1785 tasklet_schedule(&pl330->tasks); 1786 } 1787 1788 return ret; 1789 } 1790 1791 static int pl330_chan_ctrl(void *ch_id, enum pl330_chan_op op) 1792 { 1793 struct pl330_thread *thrd = ch_id; 1794 struct pl330_dmac *pl330; 1795 unsigned long flags; 1796 int ret = 0, active; 1797 1798 if (!thrd || thrd->free || thrd->dmac->state == DYING) 1799 return -EINVAL; 1800 1801 pl330 = thrd->dmac; 1802 active = thrd->req_running; 1803 1804 spin_lock_irqsave(&pl330->lock, flags); 1805 1806 switch (op) { 1807 case PL330_OP_FLUSH: 1808 /* Make sure the channel is stopped */ 1809 _stop(thrd); 1810 1811 thrd->req[0].r = NULL; 1812 thrd->req[1].r = NULL; 1813 mark_free(thrd, 0); 1814 mark_free(thrd, 1); 1815 break; 1816 1817 case PL330_OP_ABORT: 1818 /* Make sure the channel is stopped */ 1819 _stop(thrd); 1820 1821 /* ABORT is only for the active req */ 1822 if (active == -1) 1823 break; 1824 1825 thrd->req[active].r = NULL; 1826 mark_free(thrd, active); 1827 1828 /* Start the next */ 1829 case PL330_OP_START: 1830 if ((active == -1) && !_start(thrd)) 1831 ret = -EIO; 1832 break; 1833 1834 default: 1835 ret = -EINVAL; 1836 } 1837 1838 spin_unlock_irqrestore(&pl330->lock, flags); 1839 return ret; 1840 } 1841 1842 /* Reserve an event */ 1843 static inline int _alloc_event(struct pl330_thread *thrd) 1844 { 1845 struct pl330_dmac *pl330 = thrd->dmac; 1846 struct pl330_info *pi = pl330->pinfo; 1847 int ev; 1848 1849 for (ev = 0; ev < pi->pcfg.num_events; ev++) 1850 if (pl330->events[ev] == -1) { 1851 pl330->events[ev] = thrd->id; 1852 return ev; 1853 } 1854 1855 return -1; 1856 } 1857 1858 static bool _chan_ns(const struct pl330_info *pi, int i) 1859 { 1860 return pi->pcfg.irq_ns & (1 << i); 1861 } 1862 1863 /* Upon success, returns IdentityToken for the 1864 * allocated channel, NULL otherwise. 1865 */ 1866 static void *pl330_request_channel(const struct pl330_info *pi) 1867 { 1868 struct pl330_thread *thrd = NULL; 1869 struct pl330_dmac *pl330; 1870 unsigned long flags; 1871 int chans, i; 1872 1873 if (!pi || !pi->pl330_data) 1874 return NULL; 1875 1876 pl330 = pi->pl330_data; 1877 1878 if (pl330->state == DYING) 1879 return NULL; 1880 1881 chans = pi->pcfg.num_chan; 1882 1883 spin_lock_irqsave(&pl330->lock, flags); 1884 1885 for (i = 0; i < chans; i++) { 1886 thrd = &pl330->channels[i]; 1887 if ((thrd->free) && (!_manager_ns(thrd) || 1888 _chan_ns(pi, i))) { 1889 thrd->ev = _alloc_event(thrd); 1890 if (thrd->ev >= 0) { 1891 thrd->free = false; 1892 thrd->lstenq = 1; 1893 thrd->req[0].r = NULL; 1894 mark_free(thrd, 0); 1895 thrd->req[1].r = NULL; 1896 mark_free(thrd, 1); 1897 break; 1898 } 1899 } 1900 thrd = NULL; 1901 } 1902 1903 spin_unlock_irqrestore(&pl330->lock, flags); 1904 1905 return thrd; 1906 } 1907 1908 /* Release an event */ 1909 static inline void _free_event(struct pl330_thread *thrd, int ev) 1910 { 1911 struct pl330_dmac *pl330 = thrd->dmac; 1912 struct pl330_info *pi = pl330->pinfo; 1913 1914 /* If the event is valid and was held by the thread */ 1915 if (ev >= 0 && ev < pi->pcfg.num_events 1916 && pl330->events[ev] == thrd->id) 1917 pl330->events[ev] = -1; 1918 } 1919 1920 static void pl330_release_channel(void *ch_id) 1921 { 1922 struct pl330_thread *thrd = ch_id; 1923 struct pl330_dmac *pl330; 1924 unsigned long flags; 1925 1926 if (!thrd || thrd->free) 1927 return; 1928 1929 _stop(thrd); 1930 1931 _callback(thrd->req[1 - thrd->lstenq].r, PL330_ERR_ABORT); 1932 _callback(thrd->req[thrd->lstenq].r, PL330_ERR_ABORT); 1933 1934 pl330 = thrd->dmac; 1935 1936 spin_lock_irqsave(&pl330->lock, flags); 1937 _free_event(thrd, thrd->ev); 1938 thrd->free = true; 1939 spin_unlock_irqrestore(&pl330->lock, flags); 1940 } 1941 1942 /* Initialize the structure for PL330 configuration, that can be used 1943 * by the client driver the make best use of the DMAC 1944 */ 1945 static void read_dmac_config(struct pl330_info *pi) 1946 { 1947 void __iomem *regs = pi->base; 1948 u32 val; 1949 1950 val = readl(regs + CRD) >> CRD_DATA_WIDTH_SHIFT; 1951 val &= CRD_DATA_WIDTH_MASK; 1952 pi->pcfg.data_bus_width = 8 * (1 << val); 1953 1954 val = readl(regs + CRD) >> CRD_DATA_BUFF_SHIFT; 1955 val &= CRD_DATA_BUFF_MASK; 1956 pi->pcfg.data_buf_dep = val + 1; 1957 1958 val = readl(regs + CR0) >> CR0_NUM_CHANS_SHIFT; 1959 val &= CR0_NUM_CHANS_MASK; 1960 val += 1; 1961 pi->pcfg.num_chan = val; 1962 1963 val = readl(regs + CR0); 1964 if (val & CR0_PERIPH_REQ_SET) { 1965 val = (val >> CR0_NUM_PERIPH_SHIFT) & CR0_NUM_PERIPH_MASK; 1966 val += 1; 1967 pi->pcfg.num_peri = val; 1968 pi->pcfg.peri_ns = readl(regs + CR4); 1969 } else { 1970 pi->pcfg.num_peri = 0; 1971 } 1972 1973 val = readl(regs + CR0); 1974 if (val & CR0_BOOT_MAN_NS) 1975 pi->pcfg.mode |= DMAC_MODE_NS; 1976 else 1977 pi->pcfg.mode &= ~DMAC_MODE_NS; 1978 1979 val = readl(regs + CR0) >> CR0_NUM_EVENTS_SHIFT; 1980 val &= CR0_NUM_EVENTS_MASK; 1981 val += 1; 1982 pi->pcfg.num_events = val; 1983 1984 pi->pcfg.irq_ns = readl(regs + CR3); 1985 1986 pi->pcfg.periph_id = get_id(pi, PERIPH_ID); 1987 pi->pcfg.pcell_id = get_id(pi, PCELL_ID); 1988 } 1989 1990 static inline void _reset_thread(struct pl330_thread *thrd) 1991 { 1992 struct pl330_dmac *pl330 = thrd->dmac; 1993 struct pl330_info *pi = pl330->pinfo; 1994 1995 thrd->req[0].mc_cpu = pl330->mcode_cpu 1996 + (thrd->id * pi->mcbufsz); 1997 thrd->req[0].mc_bus = pl330->mcode_bus 1998 + (thrd->id * pi->mcbufsz); 1999 thrd->req[0].r = NULL; 2000 mark_free(thrd, 0); 2001 2002 thrd->req[1].mc_cpu = thrd->req[0].mc_cpu 2003 + pi->mcbufsz / 2; 2004 thrd->req[1].mc_bus = thrd->req[0].mc_bus 2005 + pi->mcbufsz / 2; 2006 thrd->req[1].r = NULL; 2007 mark_free(thrd, 1); 2008 } 2009 2010 static int dmac_alloc_threads(struct pl330_dmac *pl330) 2011 { 2012 struct pl330_info *pi = pl330->pinfo; 2013 int chans = pi->pcfg.num_chan; 2014 struct pl330_thread *thrd; 2015 int i; 2016 2017 /* Allocate 1 Manager and 'chans' Channel threads */ 2018 pl330->channels = kzalloc((1 + chans) * sizeof(*thrd), 2019 GFP_KERNEL); 2020 if (!pl330->channels) 2021 return -ENOMEM; 2022 2023 /* Init Channel threads */ 2024 for (i = 0; i < chans; i++) { 2025 thrd = &pl330->channels[i]; 2026 thrd->id = i; 2027 thrd->dmac = pl330; 2028 _reset_thread(thrd); 2029 thrd->free = true; 2030 } 2031 2032 /* MANAGER is indexed at the end */ 2033 thrd = &pl330->channels[chans]; 2034 thrd->id = chans; 2035 thrd->dmac = pl330; 2036 thrd->free = false; 2037 pl330->manager = thrd; 2038 2039 return 0; 2040 } 2041 2042 static int dmac_alloc_resources(struct pl330_dmac *pl330) 2043 { 2044 struct pl330_info *pi = pl330->pinfo; 2045 int chans = pi->pcfg.num_chan; 2046 int ret; 2047 2048 /* 2049 * Alloc MicroCode buffer for 'chans' Channel threads. 2050 * A channel's buffer offset is (Channel_Id * MCODE_BUFF_PERCHAN) 2051 */ 2052 pl330->mcode_cpu = dma_alloc_coherent(pi->dev, 2053 chans * pi->mcbufsz, 2054 &pl330->mcode_bus, GFP_KERNEL); 2055 if (!pl330->mcode_cpu) { 2056 dev_err(pi->dev, "%s:%d Can't allocate memory!\n", 2057 __func__, __LINE__); 2058 return -ENOMEM; 2059 } 2060 2061 ret = dmac_alloc_threads(pl330); 2062 if (ret) { 2063 dev_err(pi->dev, "%s:%d Can't to create channels for DMAC!\n", 2064 __func__, __LINE__); 2065 dma_free_coherent(pi->dev, 2066 chans * pi->mcbufsz, 2067 pl330->mcode_cpu, pl330->mcode_bus); 2068 return ret; 2069 } 2070 2071 return 0; 2072 } 2073 2074 static int pl330_add(struct pl330_info *pi) 2075 { 2076 struct pl330_dmac *pl330; 2077 void __iomem *regs; 2078 int i, ret; 2079 2080 if (!pi || !pi->dev) 2081 return -EINVAL; 2082 2083 /* If already added */ 2084 if (pi->pl330_data) 2085 return -EINVAL; 2086 2087 /* 2088 * If the SoC can perform reset on the DMAC, then do it 2089 * before reading its configuration. 2090 */ 2091 if (pi->dmac_reset) 2092 pi->dmac_reset(pi); 2093 2094 regs = pi->base; 2095 2096 /* Check if we can handle this DMAC */ 2097 if ((get_id(pi, PERIPH_ID) & 0xfffff) != PERIPH_ID_VAL 2098 || get_id(pi, PCELL_ID) != PCELL_ID_VAL) { 2099 dev_err(pi->dev, "PERIPH_ID 0x%x, PCELL_ID 0x%x !\n", 2100 get_id(pi, PERIPH_ID), get_id(pi, PCELL_ID)); 2101 return -EINVAL; 2102 } 2103 2104 /* Read the configuration of the DMAC */ 2105 read_dmac_config(pi); 2106 2107 if (pi->pcfg.num_events == 0) { 2108 dev_err(pi->dev, "%s:%d Can't work without events!\n", 2109 __func__, __LINE__); 2110 return -EINVAL; 2111 } 2112 2113 pl330 = kzalloc(sizeof(*pl330), GFP_KERNEL); 2114 if (!pl330) { 2115 dev_err(pi->dev, "%s:%d Can't allocate memory!\n", 2116 __func__, __LINE__); 2117 return -ENOMEM; 2118 } 2119 2120 /* Assign the info structure and private data */ 2121 pl330->pinfo = pi; 2122 pi->pl330_data = pl330; 2123 2124 spin_lock_init(&pl330->lock); 2125 2126 INIT_LIST_HEAD(&pl330->req_done); 2127 2128 /* Use default MC buffer size if not provided */ 2129 if (!pi->mcbufsz) 2130 pi->mcbufsz = MCODE_BUFF_PER_REQ * 2; 2131 2132 /* Mark all events as free */ 2133 for (i = 0; i < pi->pcfg.num_events; i++) 2134 pl330->events[i] = -1; 2135 2136 /* Allocate resources needed by the DMAC */ 2137 ret = dmac_alloc_resources(pl330); 2138 if (ret) { 2139 dev_err(pi->dev, "Unable to create channels for DMAC\n"); 2140 kfree(pl330); 2141 return ret; 2142 } 2143 2144 tasklet_init(&pl330->tasks, pl330_dotask, (unsigned long) pl330); 2145 2146 pl330->state = INIT; 2147 2148 return 0; 2149 } 2150 2151 static int dmac_free_threads(struct pl330_dmac *pl330) 2152 { 2153 struct pl330_info *pi = pl330->pinfo; 2154 int chans = pi->pcfg.num_chan; 2155 struct pl330_thread *thrd; 2156 int i; 2157 2158 /* Release Channel threads */ 2159 for (i = 0; i < chans; i++) { 2160 thrd = &pl330->channels[i]; 2161 pl330_release_channel((void *)thrd); 2162 } 2163 2164 /* Free memory */ 2165 kfree(pl330->channels); 2166 2167 return 0; 2168 } 2169 2170 static void dmac_free_resources(struct pl330_dmac *pl330) 2171 { 2172 struct pl330_info *pi = pl330->pinfo; 2173 int chans = pi->pcfg.num_chan; 2174 2175 dmac_free_threads(pl330); 2176 2177 dma_free_coherent(pi->dev, chans * pi->mcbufsz, 2178 pl330->mcode_cpu, pl330->mcode_bus); 2179 } 2180 2181 static void pl330_del(struct pl330_info *pi) 2182 { 2183 struct pl330_dmac *pl330; 2184 2185 if (!pi || !pi->pl330_data) 2186 return; 2187 2188 pl330 = pi->pl330_data; 2189 2190 pl330->state = UNINIT; 2191 2192 tasklet_kill(&pl330->tasks); 2193 2194 /* Free DMAC resources */ 2195 dmac_free_resources(pl330); 2196 2197 kfree(pl330); 2198 pi->pl330_data = NULL; 2199 } 2200 2201 /* forward declaration */ 2202 static struct amba_driver pl330_driver; 2203 2204 static inline struct dma_pl330_chan * 2205 to_pchan(struct dma_chan *ch) 2206 { 2207 if (!ch) 2208 return NULL; 2209 2210 return container_of(ch, struct dma_pl330_chan, chan); 2211 } 2212 2213 static inline struct dma_pl330_desc * 2214 to_desc(struct dma_async_tx_descriptor *tx) 2215 { 2216 return container_of(tx, struct dma_pl330_desc, txd); 2217 } 2218 2219 static inline void free_desc_list(struct list_head *list) 2220 { 2221 struct dma_pl330_dmac *pdmac; 2222 struct dma_pl330_desc *desc; 2223 struct dma_pl330_chan *pch = NULL; 2224 unsigned long flags; 2225 2226 /* Finish off the work list */ 2227 list_for_each_entry(desc, list, node) { 2228 dma_async_tx_callback callback; 2229 void *param; 2230 2231 /* All desc in a list belong to same channel */ 2232 pch = desc->pchan; 2233 callback = desc->txd.callback; 2234 param = desc->txd.callback_param; 2235 2236 if (callback) 2237 callback(param); 2238 2239 desc->pchan = NULL; 2240 } 2241 2242 /* pch will be unset if list was empty */ 2243 if (!pch) 2244 return; 2245 2246 pdmac = pch->dmac; 2247 2248 spin_lock_irqsave(&pdmac->pool_lock, flags); 2249 list_splice_tail_init(list, &pdmac->desc_pool); 2250 spin_unlock_irqrestore(&pdmac->pool_lock, flags); 2251 } 2252 2253 static inline void handle_cyclic_desc_list(struct list_head *list) 2254 { 2255 struct dma_pl330_desc *desc; 2256 struct dma_pl330_chan *pch = NULL; 2257 unsigned long flags; 2258 2259 list_for_each_entry(desc, list, node) { 2260 dma_async_tx_callback callback; 2261 2262 /* Change status to reload it */ 2263 desc->status = PREP; 2264 pch = desc->pchan; 2265 callback = desc->txd.callback; 2266 if (callback) 2267 callback(desc->txd.callback_param); 2268 } 2269 2270 /* pch will be unset if list was empty */ 2271 if (!pch) 2272 return; 2273 2274 spin_lock_irqsave(&pch->lock, flags); 2275 list_splice_tail_init(list, &pch->work_list); 2276 spin_unlock_irqrestore(&pch->lock, flags); 2277 } 2278 2279 static inline void fill_queue(struct dma_pl330_chan *pch) 2280 { 2281 struct dma_pl330_desc *desc; 2282 int ret; 2283 2284 list_for_each_entry(desc, &pch->work_list, node) { 2285 2286 /* If already submitted */ 2287 if (desc->status == BUSY) 2288 break; 2289 2290 ret = pl330_submit_req(pch->pl330_chid, 2291 &desc->req); 2292 if (!ret) { 2293 desc->status = BUSY; 2294 break; 2295 } else if (ret == -EAGAIN) { 2296 /* QFull or DMAC Dying */ 2297 break; 2298 } else { 2299 /* Unacceptable request */ 2300 desc->status = DONE; 2301 dev_err(pch->dmac->pif.dev, "%s:%d Bad Desc(%d)\n", 2302 __func__, __LINE__, desc->txd.cookie); 2303 tasklet_schedule(&pch->task); 2304 } 2305 } 2306 } 2307 2308 static void pl330_tasklet(unsigned long data) 2309 { 2310 struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data; 2311 struct dma_pl330_desc *desc, *_dt; 2312 unsigned long flags; 2313 LIST_HEAD(list); 2314 2315 spin_lock_irqsave(&pch->lock, flags); 2316 2317 /* Pick up ripe tomatoes */ 2318 list_for_each_entry_safe(desc, _dt, &pch->work_list, node) 2319 if (desc->status == DONE) { 2320 if (!pch->cyclic) 2321 dma_cookie_complete(&desc->txd); 2322 list_move_tail(&desc->node, &list); 2323 } 2324 2325 /* Try to submit a req imm. next to the last completed cookie */ 2326 fill_queue(pch); 2327 2328 /* Make sure the PL330 Channel thread is active */ 2329 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_START); 2330 2331 spin_unlock_irqrestore(&pch->lock, flags); 2332 2333 if (pch->cyclic) 2334 handle_cyclic_desc_list(&list); 2335 else 2336 free_desc_list(&list); 2337 } 2338 2339 static void dma_pl330_rqcb(void *token, enum pl330_op_err err) 2340 { 2341 struct dma_pl330_desc *desc = token; 2342 struct dma_pl330_chan *pch = desc->pchan; 2343 unsigned long flags; 2344 2345 /* If desc aborted */ 2346 if (!pch) 2347 return; 2348 2349 spin_lock_irqsave(&pch->lock, flags); 2350 2351 desc->status = DONE; 2352 2353 spin_unlock_irqrestore(&pch->lock, flags); 2354 2355 tasklet_schedule(&pch->task); 2356 } 2357 2358 bool pl330_filter(struct dma_chan *chan, void *param) 2359 { 2360 u8 *peri_id; 2361 2362 if (chan->device->dev->driver != &pl330_driver.drv) 2363 return false; 2364 2365 #ifdef CONFIG_OF 2366 if (chan->device->dev->of_node) { 2367 const __be32 *prop_value; 2368 phandle phandle; 2369 struct device_node *node; 2370 2371 prop_value = ((struct property *)param)->value; 2372 phandle = be32_to_cpup(prop_value++); 2373 node = of_find_node_by_phandle(phandle); 2374 return ((chan->private == node) && 2375 (chan->chan_id == be32_to_cpup(prop_value))); 2376 } 2377 #endif 2378 2379 peri_id = chan->private; 2380 return *peri_id == (unsigned)param; 2381 } 2382 EXPORT_SYMBOL(pl330_filter); 2383 2384 static int pl330_alloc_chan_resources(struct dma_chan *chan) 2385 { 2386 struct dma_pl330_chan *pch = to_pchan(chan); 2387 struct dma_pl330_dmac *pdmac = pch->dmac; 2388 unsigned long flags; 2389 2390 spin_lock_irqsave(&pch->lock, flags); 2391 2392 dma_cookie_init(chan); 2393 pch->cyclic = false; 2394 2395 pch->pl330_chid = pl330_request_channel(&pdmac->pif); 2396 if (!pch->pl330_chid) { 2397 spin_unlock_irqrestore(&pch->lock, flags); 2398 return 0; 2399 } 2400 2401 tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch); 2402 2403 spin_unlock_irqrestore(&pch->lock, flags); 2404 2405 return 1; 2406 } 2407 2408 static int pl330_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, unsigned long arg) 2409 { 2410 struct dma_pl330_chan *pch = to_pchan(chan); 2411 struct dma_pl330_desc *desc, *_dt; 2412 unsigned long flags; 2413 struct dma_pl330_dmac *pdmac = pch->dmac; 2414 struct dma_slave_config *slave_config; 2415 LIST_HEAD(list); 2416 2417 switch (cmd) { 2418 case DMA_TERMINATE_ALL: 2419 spin_lock_irqsave(&pch->lock, flags); 2420 2421 /* FLUSH the PL330 Channel thread */ 2422 pl330_chan_ctrl(pch->pl330_chid, PL330_OP_FLUSH); 2423 2424 /* Mark all desc done */ 2425 list_for_each_entry_safe(desc, _dt, &pch->work_list , node) { 2426 desc->status = DONE; 2427 list_move_tail(&desc->node, &list); 2428 } 2429 2430 list_splice_tail_init(&list, &pdmac->desc_pool); 2431 spin_unlock_irqrestore(&pch->lock, flags); 2432 break; 2433 case DMA_SLAVE_CONFIG: 2434 slave_config = (struct dma_slave_config *)arg; 2435 2436 if (slave_config->direction == DMA_MEM_TO_DEV) { 2437 if (slave_config->dst_addr) 2438 pch->fifo_addr = slave_config->dst_addr; 2439 if (slave_config->dst_addr_width) 2440 pch->burst_sz = __ffs(slave_config->dst_addr_width); 2441 if (slave_config->dst_maxburst) 2442 pch->burst_len = slave_config->dst_maxburst; 2443 } else if (slave_config->direction == DMA_DEV_TO_MEM) { 2444 if (slave_config->src_addr) 2445 pch->fifo_addr = slave_config->src_addr; 2446 if (slave_config->src_addr_width) 2447 pch->burst_sz = __ffs(slave_config->src_addr_width); 2448 if (slave_config->src_maxburst) 2449 pch->burst_len = slave_config->src_maxburst; 2450 } 2451 break; 2452 default: 2453 dev_err(pch->dmac->pif.dev, "Not supported command.\n"); 2454 return -ENXIO; 2455 } 2456 2457 return 0; 2458 } 2459 2460 static void pl330_free_chan_resources(struct dma_chan *chan) 2461 { 2462 struct dma_pl330_chan *pch = to_pchan(chan); 2463 unsigned long flags; 2464 2465 spin_lock_irqsave(&pch->lock, flags); 2466 2467 tasklet_kill(&pch->task); 2468 2469 pl330_release_channel(pch->pl330_chid); 2470 pch->pl330_chid = NULL; 2471 2472 if (pch->cyclic) 2473 list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool); 2474 2475 spin_unlock_irqrestore(&pch->lock, flags); 2476 } 2477 2478 static enum dma_status 2479 pl330_tx_status(struct dma_chan *chan, dma_cookie_t cookie, 2480 struct dma_tx_state *txstate) 2481 { 2482 return dma_cookie_status(chan, cookie, txstate); 2483 } 2484 2485 static void pl330_issue_pending(struct dma_chan *chan) 2486 { 2487 pl330_tasklet((unsigned long) to_pchan(chan)); 2488 } 2489 2490 /* 2491 * We returned the last one of the circular list of descriptor(s) 2492 * from prep_xxx, so the argument to submit corresponds to the last 2493 * descriptor of the list. 2494 */ 2495 static dma_cookie_t pl330_tx_submit(struct dma_async_tx_descriptor *tx) 2496 { 2497 struct dma_pl330_desc *desc, *last = to_desc(tx); 2498 struct dma_pl330_chan *pch = to_pchan(tx->chan); 2499 dma_cookie_t cookie; 2500 unsigned long flags; 2501 2502 spin_lock_irqsave(&pch->lock, flags); 2503 2504 /* Assign cookies to all nodes */ 2505 while (!list_empty(&last->node)) { 2506 desc = list_entry(last->node.next, struct dma_pl330_desc, node); 2507 2508 dma_cookie_assign(&desc->txd); 2509 2510 list_move_tail(&desc->node, &pch->work_list); 2511 } 2512 2513 cookie = dma_cookie_assign(&last->txd); 2514 list_add_tail(&last->node, &pch->work_list); 2515 spin_unlock_irqrestore(&pch->lock, flags); 2516 2517 return cookie; 2518 } 2519 2520 static inline void _init_desc(struct dma_pl330_desc *desc) 2521 { 2522 desc->pchan = NULL; 2523 desc->req.x = &desc->px; 2524 desc->req.token = desc; 2525 desc->rqcfg.swap = SWAP_NO; 2526 desc->rqcfg.privileged = 0; 2527 desc->rqcfg.insnaccess = 0; 2528 desc->rqcfg.scctl = SCCTRL0; 2529 desc->rqcfg.dcctl = DCCTRL0; 2530 desc->req.cfg = &desc->rqcfg; 2531 desc->req.xfer_cb = dma_pl330_rqcb; 2532 desc->txd.tx_submit = pl330_tx_submit; 2533 2534 INIT_LIST_HEAD(&desc->node); 2535 } 2536 2537 /* Returns the number of descriptors added to the DMAC pool */ 2538 static int add_desc(struct dma_pl330_dmac *pdmac, gfp_t flg, int count) 2539 { 2540 struct dma_pl330_desc *desc; 2541 unsigned long flags; 2542 int i; 2543 2544 if (!pdmac) 2545 return 0; 2546 2547 desc = kmalloc(count * sizeof(*desc), flg); 2548 if (!desc) 2549 return 0; 2550 2551 spin_lock_irqsave(&pdmac->pool_lock, flags); 2552 2553 for (i = 0; i < count; i++) { 2554 _init_desc(&desc[i]); 2555 list_add_tail(&desc[i].node, &pdmac->desc_pool); 2556 } 2557 2558 spin_unlock_irqrestore(&pdmac->pool_lock, flags); 2559 2560 return count; 2561 } 2562 2563 static struct dma_pl330_desc * 2564 pluck_desc(struct dma_pl330_dmac *pdmac) 2565 { 2566 struct dma_pl330_desc *desc = NULL; 2567 unsigned long flags; 2568 2569 if (!pdmac) 2570 return NULL; 2571 2572 spin_lock_irqsave(&pdmac->pool_lock, flags); 2573 2574 if (!list_empty(&pdmac->desc_pool)) { 2575 desc = list_entry(pdmac->desc_pool.next, 2576 struct dma_pl330_desc, node); 2577 2578 list_del_init(&desc->node); 2579 2580 desc->status = PREP; 2581 desc->txd.callback = NULL; 2582 } 2583 2584 spin_unlock_irqrestore(&pdmac->pool_lock, flags); 2585 2586 return desc; 2587 } 2588 2589 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch) 2590 { 2591 struct dma_pl330_dmac *pdmac = pch->dmac; 2592 u8 *peri_id = pch->chan.private; 2593 struct dma_pl330_desc *desc; 2594 2595 /* Pluck one desc from the pool of DMAC */ 2596 desc = pluck_desc(pdmac); 2597 2598 /* If the DMAC pool is empty, alloc new */ 2599 if (!desc) { 2600 if (!add_desc(pdmac, GFP_ATOMIC, 1)) 2601 return NULL; 2602 2603 /* Try again */ 2604 desc = pluck_desc(pdmac); 2605 if (!desc) { 2606 dev_err(pch->dmac->pif.dev, 2607 "%s:%d ALERT!\n", __func__, __LINE__); 2608 return NULL; 2609 } 2610 } 2611 2612 /* Initialize the descriptor */ 2613 desc->pchan = pch; 2614 desc->txd.cookie = 0; 2615 async_tx_ack(&desc->txd); 2616 2617 desc->req.peri = peri_id ? pch->chan.chan_id : 0; 2618 desc->rqcfg.pcfg = &pch->dmac->pif.pcfg; 2619 2620 dma_async_tx_descriptor_init(&desc->txd, &pch->chan); 2621 2622 return desc; 2623 } 2624 2625 static inline void fill_px(struct pl330_xfer *px, 2626 dma_addr_t dst, dma_addr_t src, size_t len) 2627 { 2628 px->next = NULL; 2629 px->bytes = len; 2630 px->dst_addr = dst; 2631 px->src_addr = src; 2632 } 2633 2634 static struct dma_pl330_desc * 2635 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst, 2636 dma_addr_t src, size_t len) 2637 { 2638 struct dma_pl330_desc *desc = pl330_get_desc(pch); 2639 2640 if (!desc) { 2641 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n", 2642 __func__, __LINE__); 2643 return NULL; 2644 } 2645 2646 /* 2647 * Ideally we should lookout for reqs bigger than 2648 * those that can be programmed with 256 bytes of 2649 * MC buffer, but considering a req size is seldom 2650 * going to be word-unaligned and more than 200MB, 2651 * we take it easy. 2652 * Also, should the limit is reached we'd rather 2653 * have the platform increase MC buffer size than 2654 * complicating this API driver. 2655 */ 2656 fill_px(&desc->px, dst, src, len); 2657 2658 return desc; 2659 } 2660 2661 /* Call after fixing burst size */ 2662 static inline int get_burst_len(struct dma_pl330_desc *desc, size_t len) 2663 { 2664 struct dma_pl330_chan *pch = desc->pchan; 2665 struct pl330_info *pi = &pch->dmac->pif; 2666 int burst_len; 2667 2668 burst_len = pi->pcfg.data_bus_width / 8; 2669 burst_len *= pi->pcfg.data_buf_dep; 2670 burst_len >>= desc->rqcfg.brst_size; 2671 2672 /* src/dst_burst_len can't be more than 16 */ 2673 if (burst_len > 16) 2674 burst_len = 16; 2675 2676 while (burst_len > 1) { 2677 if (!(len % (burst_len << desc->rqcfg.brst_size))) 2678 break; 2679 burst_len--; 2680 } 2681 2682 return burst_len; 2683 } 2684 2685 static struct dma_async_tx_descriptor *pl330_prep_dma_cyclic( 2686 struct dma_chan *chan, dma_addr_t dma_addr, size_t len, 2687 size_t period_len, enum dma_transfer_direction direction, 2688 void *context) 2689 { 2690 struct dma_pl330_desc *desc; 2691 struct dma_pl330_chan *pch = to_pchan(chan); 2692 dma_addr_t dst; 2693 dma_addr_t src; 2694 2695 desc = pl330_get_desc(pch); 2696 if (!desc) { 2697 dev_err(pch->dmac->pif.dev, "%s:%d Unable to fetch desc\n", 2698 __func__, __LINE__); 2699 return NULL; 2700 } 2701 2702 switch (direction) { 2703 case DMA_MEM_TO_DEV: 2704 desc->rqcfg.src_inc = 1; 2705 desc->rqcfg.dst_inc = 0; 2706 desc->req.rqtype = MEMTODEV; 2707 src = dma_addr; 2708 dst = pch->fifo_addr; 2709 break; 2710 case DMA_DEV_TO_MEM: 2711 desc->rqcfg.src_inc = 0; 2712 desc->rqcfg.dst_inc = 1; 2713 desc->req.rqtype = DEVTOMEM; 2714 src = pch->fifo_addr; 2715 dst = dma_addr; 2716 break; 2717 default: 2718 dev_err(pch->dmac->pif.dev, "%s:%d Invalid dma direction\n", 2719 __func__, __LINE__); 2720 return NULL; 2721 } 2722 2723 desc->rqcfg.brst_size = pch->burst_sz; 2724 desc->rqcfg.brst_len = 1; 2725 2726 pch->cyclic = true; 2727 2728 fill_px(&desc->px, dst, src, period_len); 2729 2730 return &desc->txd; 2731 } 2732 2733 static struct dma_async_tx_descriptor * 2734 pl330_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dst, 2735 dma_addr_t src, size_t len, unsigned long flags) 2736 { 2737 struct dma_pl330_desc *desc; 2738 struct dma_pl330_chan *pch = to_pchan(chan); 2739 struct pl330_info *pi; 2740 int burst; 2741 2742 if (unlikely(!pch || !len)) 2743 return NULL; 2744 2745 pi = &pch->dmac->pif; 2746 2747 desc = __pl330_prep_dma_memcpy(pch, dst, src, len); 2748 if (!desc) 2749 return NULL; 2750 2751 desc->rqcfg.src_inc = 1; 2752 desc->rqcfg.dst_inc = 1; 2753 desc->req.rqtype = MEMTOMEM; 2754 2755 /* Select max possible burst size */ 2756 burst = pi->pcfg.data_bus_width / 8; 2757 2758 while (burst > 1) { 2759 if (!(len % burst)) 2760 break; 2761 burst /= 2; 2762 } 2763 2764 desc->rqcfg.brst_size = 0; 2765 while (burst != (1 << desc->rqcfg.brst_size)) 2766 desc->rqcfg.brst_size++; 2767 2768 desc->rqcfg.brst_len = get_burst_len(desc, len); 2769 2770 desc->txd.flags = flags; 2771 2772 return &desc->txd; 2773 } 2774 2775 static struct dma_async_tx_descriptor * 2776 pl330_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl, 2777 unsigned int sg_len, enum dma_transfer_direction direction, 2778 unsigned long flg, void *context) 2779 { 2780 struct dma_pl330_desc *first, *desc = NULL; 2781 struct dma_pl330_chan *pch = to_pchan(chan); 2782 struct scatterlist *sg; 2783 unsigned long flags; 2784 int i; 2785 dma_addr_t addr; 2786 2787 if (unlikely(!pch || !sgl || !sg_len)) 2788 return NULL; 2789 2790 addr = pch->fifo_addr; 2791 2792 first = NULL; 2793 2794 for_each_sg(sgl, sg, sg_len, i) { 2795 2796 desc = pl330_get_desc(pch); 2797 if (!desc) { 2798 struct dma_pl330_dmac *pdmac = pch->dmac; 2799 2800 dev_err(pch->dmac->pif.dev, 2801 "%s:%d Unable to fetch desc\n", 2802 __func__, __LINE__); 2803 if (!first) 2804 return NULL; 2805 2806 spin_lock_irqsave(&pdmac->pool_lock, flags); 2807 2808 while (!list_empty(&first->node)) { 2809 desc = list_entry(first->node.next, 2810 struct dma_pl330_desc, node); 2811 list_move_tail(&desc->node, &pdmac->desc_pool); 2812 } 2813 2814 list_move_tail(&first->node, &pdmac->desc_pool); 2815 2816 spin_unlock_irqrestore(&pdmac->pool_lock, flags); 2817 2818 return NULL; 2819 } 2820 2821 if (!first) 2822 first = desc; 2823 else 2824 list_add_tail(&desc->node, &first->node); 2825 2826 if (direction == DMA_MEM_TO_DEV) { 2827 desc->rqcfg.src_inc = 1; 2828 desc->rqcfg.dst_inc = 0; 2829 desc->req.rqtype = MEMTODEV; 2830 fill_px(&desc->px, 2831 addr, sg_dma_address(sg), sg_dma_len(sg)); 2832 } else { 2833 desc->rqcfg.src_inc = 0; 2834 desc->rqcfg.dst_inc = 1; 2835 desc->req.rqtype = DEVTOMEM; 2836 fill_px(&desc->px, 2837 sg_dma_address(sg), addr, sg_dma_len(sg)); 2838 } 2839 2840 desc->rqcfg.brst_size = pch->burst_sz; 2841 desc->rqcfg.brst_len = 1; 2842 } 2843 2844 /* Return the last desc in the chain */ 2845 desc->txd.flags = flg; 2846 return &desc->txd; 2847 } 2848 2849 static irqreturn_t pl330_irq_handler(int irq, void *data) 2850 { 2851 if (pl330_update(data)) 2852 return IRQ_HANDLED; 2853 else 2854 return IRQ_NONE; 2855 } 2856 2857 static int __devinit 2858 pl330_probe(struct amba_device *adev, const struct amba_id *id) 2859 { 2860 struct dma_pl330_platdata *pdat; 2861 struct dma_pl330_dmac *pdmac; 2862 struct dma_pl330_chan *pch; 2863 struct pl330_info *pi; 2864 struct dma_device *pd; 2865 struct resource *res; 2866 int i, ret, irq; 2867 int num_chan; 2868 2869 pdat = adev->dev.platform_data; 2870 2871 /* Allocate a new DMAC and its Channels */ 2872 pdmac = kzalloc(sizeof(*pdmac), GFP_KERNEL); 2873 if (!pdmac) { 2874 dev_err(&adev->dev, "unable to allocate mem\n"); 2875 return -ENOMEM; 2876 } 2877 2878 pi = &pdmac->pif; 2879 pi->dev = &adev->dev; 2880 pi->pl330_data = NULL; 2881 pi->mcbufsz = pdat ? pdat->mcbuf_sz : 0; 2882 2883 res = &adev->res; 2884 request_mem_region(res->start, resource_size(res), "dma-pl330"); 2885 2886 pi->base = ioremap(res->start, resource_size(res)); 2887 if (!pi->base) { 2888 ret = -ENXIO; 2889 goto probe_err1; 2890 } 2891 2892 pdmac->clk = clk_get(&adev->dev, "dma"); 2893 if (IS_ERR(pdmac->clk)) { 2894 dev_err(&adev->dev, "Cannot get operation clock.\n"); 2895 ret = -EINVAL; 2896 goto probe_err2; 2897 } 2898 2899 amba_set_drvdata(adev, pdmac); 2900 2901 #ifndef CONFIG_PM_RUNTIME 2902 /* enable dma clk */ 2903 clk_enable(pdmac->clk); 2904 #endif 2905 2906 irq = adev->irq[0]; 2907 ret = request_irq(irq, pl330_irq_handler, 0, 2908 dev_name(&adev->dev), pi); 2909 if (ret) 2910 goto probe_err3; 2911 2912 ret = pl330_add(pi); 2913 if (ret) 2914 goto probe_err4; 2915 2916 INIT_LIST_HEAD(&pdmac->desc_pool); 2917 spin_lock_init(&pdmac->pool_lock); 2918 2919 /* Create a descriptor pool of default size */ 2920 if (!add_desc(pdmac, GFP_KERNEL, NR_DEFAULT_DESC)) 2921 dev_warn(&adev->dev, "unable to allocate desc\n"); 2922 2923 pd = &pdmac->ddma; 2924 INIT_LIST_HEAD(&pd->channels); 2925 2926 /* Initialize channel parameters */ 2927 if (pdat) 2928 num_chan = max_t(int, pdat->nr_valid_peri, pi->pcfg.num_chan); 2929 else 2930 num_chan = max_t(int, pi->pcfg.num_peri, pi->pcfg.num_chan); 2931 2932 pdmac->peripherals = kzalloc(num_chan * sizeof(*pch), GFP_KERNEL); 2933 if (!pdmac->peripherals) { 2934 ret = -ENOMEM; 2935 dev_err(&adev->dev, "unable to allocate pdmac->peripherals\n"); 2936 goto probe_err5; 2937 } 2938 2939 for (i = 0; i < num_chan; i++) { 2940 pch = &pdmac->peripherals[i]; 2941 if (!adev->dev.of_node) 2942 pch->chan.private = pdat ? &pdat->peri_id[i] : NULL; 2943 else 2944 pch->chan.private = adev->dev.of_node; 2945 2946 INIT_LIST_HEAD(&pch->work_list); 2947 spin_lock_init(&pch->lock); 2948 pch->pl330_chid = NULL; 2949 pch->chan.device = pd; 2950 pch->dmac = pdmac; 2951 2952 /* Add the channel to the DMAC list */ 2953 list_add_tail(&pch->chan.device_node, &pd->channels); 2954 } 2955 2956 pd->dev = &adev->dev; 2957 if (pdat) { 2958 pd->cap_mask = pdat->cap_mask; 2959 } else { 2960 dma_cap_set(DMA_MEMCPY, pd->cap_mask); 2961 if (pi->pcfg.num_peri) { 2962 dma_cap_set(DMA_SLAVE, pd->cap_mask); 2963 dma_cap_set(DMA_CYCLIC, pd->cap_mask); 2964 } 2965 } 2966 2967 pd->device_alloc_chan_resources = pl330_alloc_chan_resources; 2968 pd->device_free_chan_resources = pl330_free_chan_resources; 2969 pd->device_prep_dma_memcpy = pl330_prep_dma_memcpy; 2970 pd->device_prep_dma_cyclic = pl330_prep_dma_cyclic; 2971 pd->device_tx_status = pl330_tx_status; 2972 pd->device_prep_slave_sg = pl330_prep_slave_sg; 2973 pd->device_control = pl330_control; 2974 pd->device_issue_pending = pl330_issue_pending; 2975 2976 ret = dma_async_device_register(pd); 2977 if (ret) { 2978 dev_err(&adev->dev, "unable to register DMAC\n"); 2979 goto probe_err5; 2980 } 2981 2982 dev_info(&adev->dev, 2983 "Loaded driver for PL330 DMAC-%d\n", adev->periphid); 2984 dev_info(&adev->dev, 2985 "\tDBUFF-%ux%ubytes Num_Chans-%u Num_Peri-%u Num_Events-%u\n", 2986 pi->pcfg.data_buf_dep, 2987 pi->pcfg.data_bus_width / 8, pi->pcfg.num_chan, 2988 pi->pcfg.num_peri, pi->pcfg.num_events); 2989 2990 return 0; 2991 2992 probe_err5: 2993 pl330_del(pi); 2994 probe_err4: 2995 free_irq(irq, pi); 2996 probe_err3: 2997 #ifndef CONFIG_PM_RUNTIME 2998 clk_disable(pdmac->clk); 2999 #endif 3000 clk_put(pdmac->clk); 3001 probe_err2: 3002 iounmap(pi->base); 3003 probe_err1: 3004 release_mem_region(res->start, resource_size(res)); 3005 kfree(pdmac); 3006 3007 return ret; 3008 } 3009 3010 static int __devexit pl330_remove(struct amba_device *adev) 3011 { 3012 struct dma_pl330_dmac *pdmac = amba_get_drvdata(adev); 3013 struct dma_pl330_chan *pch, *_p; 3014 struct pl330_info *pi; 3015 struct resource *res; 3016 int irq; 3017 3018 if (!pdmac) 3019 return 0; 3020 3021 amba_set_drvdata(adev, NULL); 3022 3023 /* Idle the DMAC */ 3024 list_for_each_entry_safe(pch, _p, &pdmac->ddma.channels, 3025 chan.device_node) { 3026 3027 /* Remove the channel */ 3028 list_del(&pch->chan.device_node); 3029 3030 /* Flush the channel */ 3031 pl330_control(&pch->chan, DMA_TERMINATE_ALL, 0); 3032 pl330_free_chan_resources(&pch->chan); 3033 } 3034 3035 pi = &pdmac->pif; 3036 3037 pl330_del(pi); 3038 3039 irq = adev->irq[0]; 3040 free_irq(irq, pi); 3041 3042 iounmap(pi->base); 3043 3044 res = &adev->res; 3045 release_mem_region(res->start, resource_size(res)); 3046 3047 #ifndef CONFIG_PM_RUNTIME 3048 clk_disable(pdmac->clk); 3049 #endif 3050 3051 kfree(pdmac); 3052 3053 return 0; 3054 } 3055 3056 static struct amba_id pl330_ids[] = { 3057 { 3058 .id = 0x00041330, 3059 .mask = 0x000fffff, 3060 }, 3061 { 0, 0 }, 3062 }; 3063 3064 MODULE_DEVICE_TABLE(amba, pl330_ids); 3065 3066 #ifdef CONFIG_PM_RUNTIME 3067 static int pl330_runtime_suspend(struct device *dev) 3068 { 3069 struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev); 3070 3071 if (!pdmac) { 3072 dev_err(dev, "failed to get dmac\n"); 3073 return -ENODEV; 3074 } 3075 3076 clk_disable(pdmac->clk); 3077 3078 return 0; 3079 } 3080 3081 static int pl330_runtime_resume(struct device *dev) 3082 { 3083 struct dma_pl330_dmac *pdmac = dev_get_drvdata(dev); 3084 3085 if (!pdmac) { 3086 dev_err(dev, "failed to get dmac\n"); 3087 return -ENODEV; 3088 } 3089 3090 clk_enable(pdmac->clk); 3091 3092 return 0; 3093 } 3094 #else 3095 #define pl330_runtime_suspend NULL 3096 #define pl330_runtime_resume NULL 3097 #endif /* CONFIG_PM_RUNTIME */ 3098 3099 static const struct dev_pm_ops pl330_pm_ops = { 3100 .runtime_suspend = pl330_runtime_suspend, 3101 .runtime_resume = pl330_runtime_resume, 3102 }; 3103 3104 static struct amba_driver pl330_driver = { 3105 .drv = { 3106 .owner = THIS_MODULE, 3107 .name = "dma-pl330", 3108 .pm = &pl330_pm_ops, 3109 }, 3110 .id_table = pl330_ids, 3111 .probe = pl330_probe, 3112 .remove = pl330_remove, 3113 }; 3114 3115 module_amba_driver(pl330_driver); 3116 3117 MODULE_AUTHOR("Jaswinder Singh <jassi.brar@samsung.com>"); 3118 MODULE_DESCRIPTION("API Driver for PL330 DMAC"); 3119 MODULE_LICENSE("GPL"); 3120