1 /* 2 * Copyright (C) Ericsson AB 2007-2008 3 * Copyright (C) ST-Ericsson SA 2008-2010 4 * Author: Per Forlin <per.forlin@stericsson.com> for ST-Ericsson 5 * Author: Jonas Aaberg <jonas.aberg@stericsson.com> for ST-Ericsson 6 * License terms: GNU General Public License (GPL) version 2 7 */ 8 9 #include <linux/kernel.h> 10 #include <linux/slab.h> 11 #include <linux/dmaengine.h> 12 #include <linux/platform_device.h> 13 #include <linux/clk.h> 14 #include <linux/delay.h> 15 #include <linux/err.h> 16 17 #include <plat/ste_dma40.h> 18 19 #include "ste_dma40_ll.h" 20 21 #define D40_NAME "dma40" 22 23 #define D40_PHY_CHAN -1 24 25 /* For masking out/in 2 bit channel positions */ 26 #define D40_CHAN_POS(chan) (2 * (chan / 2)) 27 #define D40_CHAN_POS_MASK(chan) (0x3 << D40_CHAN_POS(chan)) 28 29 /* Maximum iterations taken before giving up suspending a channel */ 30 #define D40_SUSPEND_MAX_IT 500 31 32 /* Hardware requirement on LCLA alignment */ 33 #define LCLA_ALIGNMENT 0x40000 34 35 /* Max number of links per event group */ 36 #define D40_LCLA_LINK_PER_EVENT_GRP 128 37 #define D40_LCLA_END D40_LCLA_LINK_PER_EVENT_GRP 38 39 /* Attempts before giving up to trying to get pages that are aligned */ 40 #define MAX_LCLA_ALLOC_ATTEMPTS 256 41 42 /* Bit markings for allocation map */ 43 #define D40_ALLOC_FREE (1 << 31) 44 #define D40_ALLOC_PHY (1 << 30) 45 #define D40_ALLOC_LOG_FREE 0 46 47 /* Hardware designer of the block */ 48 #define D40_HW_DESIGNER 0x8 49 50 /** 51 * enum 40_command - The different commands and/or statuses. 52 * 53 * @D40_DMA_STOP: DMA channel command STOP or status STOPPED, 54 * @D40_DMA_RUN: The DMA channel is RUNNING of the command RUN. 55 * @D40_DMA_SUSPEND_REQ: Request the DMA to SUSPEND as soon as possible. 56 * @D40_DMA_SUSPENDED: The DMA channel is SUSPENDED. 57 */ 58 enum d40_command { 59 D40_DMA_STOP = 0, 60 D40_DMA_RUN = 1, 61 D40_DMA_SUSPEND_REQ = 2, 62 D40_DMA_SUSPENDED = 3 63 }; 64 65 /** 66 * struct d40_lli_pool - Structure for keeping LLIs in memory 67 * 68 * @base: Pointer to memory area when the pre_alloc_lli's are not large 69 * enough, IE bigger than the most common case, 1 dst and 1 src. NULL if 70 * pre_alloc_lli is used. 71 * @dma_addr: DMA address, if mapped 72 * @size: The size in bytes of the memory at base or the size of pre_alloc_lli. 73 * @pre_alloc_lli: Pre allocated area for the most common case of transfers, 74 * one buffer to one buffer. 75 */ 76 struct d40_lli_pool { 77 void *base; 78 int size; 79 dma_addr_t dma_addr; 80 /* Space for dst and src, plus an extra for padding */ 81 u8 pre_alloc_lli[3 * sizeof(struct d40_phy_lli)]; 82 }; 83 84 /** 85 * struct d40_desc - A descriptor is one DMA job. 86 * 87 * @lli_phy: LLI settings for physical channel. Both src and dst= 88 * points into the lli_pool, to base if lli_len > 1 or to pre_alloc_lli if 89 * lli_len equals one. 90 * @lli_log: Same as above but for logical channels. 91 * @lli_pool: The pool with two entries pre-allocated. 92 * @lli_len: Number of llis of current descriptor. 93 * @lli_current: Number of transferred llis. 94 * @lcla_alloc: Number of LCLA entries allocated. 95 * @txd: DMA engine struct. Used for among other things for communication 96 * during a transfer. 97 * @node: List entry. 98 * @is_in_client_list: true if the client owns this descriptor. 99 * the previous one. 100 * 101 * This descriptor is used for both logical and physical transfers. 102 */ 103 struct d40_desc { 104 /* LLI physical */ 105 struct d40_phy_lli_bidir lli_phy; 106 /* LLI logical */ 107 struct d40_log_lli_bidir lli_log; 108 109 struct d40_lli_pool lli_pool; 110 int lli_len; 111 int lli_current; 112 int lcla_alloc; 113 114 struct dma_async_tx_descriptor txd; 115 struct list_head node; 116 117 bool is_in_client_list; 118 bool cyclic; 119 }; 120 121 /** 122 * struct d40_lcla_pool - LCLA pool settings and data. 123 * 124 * @base: The virtual address of LCLA. 18 bit aligned. 125 * @base_unaligned: The orignal kmalloc pointer, if kmalloc is used. 126 * This pointer is only there for clean-up on error. 127 * @pages: The number of pages needed for all physical channels. 128 * Only used later for clean-up on error 129 * @lock: Lock to protect the content in this struct. 130 * @alloc_map: big map over which LCLA entry is own by which job. 131 */ 132 struct d40_lcla_pool { 133 void *base; 134 dma_addr_t dma_addr; 135 void *base_unaligned; 136 int pages; 137 spinlock_t lock; 138 struct d40_desc **alloc_map; 139 }; 140 141 /** 142 * struct d40_phy_res - struct for handling eventlines mapped to physical 143 * channels. 144 * 145 * @lock: A lock protection this entity. 146 * @num: The physical channel number of this entity. 147 * @allocated_src: Bit mapped to show which src event line's are mapped to 148 * this physical channel. Can also be free or physically allocated. 149 * @allocated_dst: Same as for src but is dst. 150 * allocated_dst and allocated_src uses the D40_ALLOC* defines as well as 151 * event line number. 152 */ 153 struct d40_phy_res { 154 spinlock_t lock; 155 int num; 156 u32 allocated_src; 157 u32 allocated_dst; 158 }; 159 160 struct d40_base; 161 162 /** 163 * struct d40_chan - Struct that describes a channel. 164 * 165 * @lock: A spinlock to protect this struct. 166 * @log_num: The logical number, if any of this channel. 167 * @completed: Starts with 1, after first interrupt it is set to dma engine's 168 * current cookie. 169 * @pending_tx: The number of pending transfers. Used between interrupt handler 170 * and tasklet. 171 * @busy: Set to true when transfer is ongoing on this channel. 172 * @phy_chan: Pointer to physical channel which this instance runs on. If this 173 * point is NULL, then the channel is not allocated. 174 * @chan: DMA engine handle. 175 * @tasklet: Tasklet that gets scheduled from interrupt context to complete a 176 * transfer and call client callback. 177 * @client: Cliented owned descriptor list. 178 * @active: Active descriptor. 179 * @queue: Queued jobs. 180 * @dma_cfg: The client configuration of this dma channel. 181 * @configured: whether the dma_cfg configuration is valid 182 * @base: Pointer to the device instance struct. 183 * @src_def_cfg: Default cfg register setting for src. 184 * @dst_def_cfg: Default cfg register setting for dst. 185 * @log_def: Default logical channel settings. 186 * @lcla: Space for one dst src pair for logical channel transfers. 187 * @lcpa: Pointer to dst and src lcpa settings. 188 * 189 * This struct can either "be" a logical or a physical channel. 190 */ 191 struct d40_chan { 192 spinlock_t lock; 193 int log_num; 194 /* ID of the most recent completed transfer */ 195 int completed; 196 int pending_tx; 197 bool busy; 198 struct d40_phy_res *phy_chan; 199 struct dma_chan chan; 200 struct tasklet_struct tasklet; 201 struct list_head client; 202 struct list_head active; 203 struct list_head queue; 204 struct stedma40_chan_cfg dma_cfg; 205 bool configured; 206 struct d40_base *base; 207 /* Default register configurations */ 208 u32 src_def_cfg; 209 u32 dst_def_cfg; 210 struct d40_def_lcsp log_def; 211 struct d40_log_lli_full *lcpa; 212 /* Runtime reconfiguration */ 213 dma_addr_t runtime_addr; 214 enum dma_data_direction runtime_direction; 215 }; 216 217 /** 218 * struct d40_base - The big global struct, one for each probe'd instance. 219 * 220 * @interrupt_lock: Lock used to make sure one interrupt is handle a time. 221 * @execmd_lock: Lock for execute command usage since several channels share 222 * the same physical register. 223 * @dev: The device structure. 224 * @virtbase: The virtual base address of the DMA's register. 225 * @rev: silicon revision detected. 226 * @clk: Pointer to the DMA clock structure. 227 * @phy_start: Physical memory start of the DMA registers. 228 * @phy_size: Size of the DMA register map. 229 * @irq: The IRQ number. 230 * @num_phy_chans: The number of physical channels. Read from HW. This 231 * is the number of available channels for this driver, not counting "Secure 232 * mode" allocated physical channels. 233 * @num_log_chans: The number of logical channels. Calculated from 234 * num_phy_chans. 235 * @dma_both: dma_device channels that can do both memcpy and slave transfers. 236 * @dma_slave: dma_device channels that can do only do slave transfers. 237 * @dma_memcpy: dma_device channels that can do only do memcpy transfers. 238 * @log_chans: Room for all possible logical channels in system. 239 * @lookup_log_chans: Used to map interrupt number to logical channel. Points 240 * to log_chans entries. 241 * @lookup_phy_chans: Used to map interrupt number to physical channel. Points 242 * to phy_chans entries. 243 * @plat_data: Pointer to provided platform_data which is the driver 244 * configuration. 245 * @phy_res: Vector containing all physical channels. 246 * @lcla_pool: lcla pool settings and data. 247 * @lcpa_base: The virtual mapped address of LCPA. 248 * @phy_lcpa: The physical address of the LCPA. 249 * @lcpa_size: The size of the LCPA area. 250 * @desc_slab: cache for descriptors. 251 */ 252 struct d40_base { 253 spinlock_t interrupt_lock; 254 spinlock_t execmd_lock; 255 struct device *dev; 256 void __iomem *virtbase; 257 u8 rev:4; 258 struct clk *clk; 259 phys_addr_t phy_start; 260 resource_size_t phy_size; 261 int irq; 262 int num_phy_chans; 263 int num_log_chans; 264 struct dma_device dma_both; 265 struct dma_device dma_slave; 266 struct dma_device dma_memcpy; 267 struct d40_chan *phy_chans; 268 struct d40_chan *log_chans; 269 struct d40_chan **lookup_log_chans; 270 struct d40_chan **lookup_phy_chans; 271 struct stedma40_platform_data *plat_data; 272 /* Physical half channels */ 273 struct d40_phy_res *phy_res; 274 struct d40_lcla_pool lcla_pool; 275 void *lcpa_base; 276 dma_addr_t phy_lcpa; 277 resource_size_t lcpa_size; 278 struct kmem_cache *desc_slab; 279 }; 280 281 /** 282 * struct d40_interrupt_lookup - lookup table for interrupt handler 283 * 284 * @src: Interrupt mask register. 285 * @clr: Interrupt clear register. 286 * @is_error: true if this is an error interrupt. 287 * @offset: start delta in the lookup_log_chans in d40_base. If equals to 288 * D40_PHY_CHAN, the lookup_phy_chans shall be used instead. 289 */ 290 struct d40_interrupt_lookup { 291 u32 src; 292 u32 clr; 293 bool is_error; 294 int offset; 295 }; 296 297 /** 298 * struct d40_reg_val - simple lookup struct 299 * 300 * @reg: The register. 301 * @val: The value that belongs to the register in reg. 302 */ 303 struct d40_reg_val { 304 unsigned int reg; 305 unsigned int val; 306 }; 307 308 static struct device *chan2dev(struct d40_chan *d40c) 309 { 310 return &d40c->chan.dev->device; 311 } 312 313 static bool chan_is_physical(struct d40_chan *chan) 314 { 315 return chan->log_num == D40_PHY_CHAN; 316 } 317 318 static bool chan_is_logical(struct d40_chan *chan) 319 { 320 return !chan_is_physical(chan); 321 } 322 323 static void __iomem *chan_base(struct d40_chan *chan) 324 { 325 return chan->base->virtbase + D40_DREG_PCBASE + 326 chan->phy_chan->num * D40_DREG_PCDELTA; 327 } 328 329 #define d40_err(dev, format, arg...) \ 330 dev_err(dev, "[%s] " format, __func__, ## arg) 331 332 #define chan_err(d40c, format, arg...) \ 333 d40_err(chan2dev(d40c), format, ## arg) 334 335 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d, 336 int lli_len) 337 { 338 bool is_log = chan_is_logical(d40c); 339 u32 align; 340 void *base; 341 342 if (is_log) 343 align = sizeof(struct d40_log_lli); 344 else 345 align = sizeof(struct d40_phy_lli); 346 347 if (lli_len == 1) { 348 base = d40d->lli_pool.pre_alloc_lli; 349 d40d->lli_pool.size = sizeof(d40d->lli_pool.pre_alloc_lli); 350 d40d->lli_pool.base = NULL; 351 } else { 352 d40d->lli_pool.size = lli_len * 2 * align; 353 354 base = kmalloc(d40d->lli_pool.size + align, GFP_NOWAIT); 355 d40d->lli_pool.base = base; 356 357 if (d40d->lli_pool.base == NULL) 358 return -ENOMEM; 359 } 360 361 if (is_log) { 362 d40d->lli_log.src = PTR_ALIGN(base, align); 363 d40d->lli_log.dst = d40d->lli_log.src + lli_len; 364 365 d40d->lli_pool.dma_addr = 0; 366 } else { 367 d40d->lli_phy.src = PTR_ALIGN(base, align); 368 d40d->lli_phy.dst = d40d->lli_phy.src + lli_len; 369 370 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev, 371 d40d->lli_phy.src, 372 d40d->lli_pool.size, 373 DMA_TO_DEVICE); 374 375 if (dma_mapping_error(d40c->base->dev, 376 d40d->lli_pool.dma_addr)) { 377 kfree(d40d->lli_pool.base); 378 d40d->lli_pool.base = NULL; 379 d40d->lli_pool.dma_addr = 0; 380 return -ENOMEM; 381 } 382 } 383 384 return 0; 385 } 386 387 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d) 388 { 389 if (d40d->lli_pool.dma_addr) 390 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr, 391 d40d->lli_pool.size, DMA_TO_DEVICE); 392 393 kfree(d40d->lli_pool.base); 394 d40d->lli_pool.base = NULL; 395 d40d->lli_pool.size = 0; 396 d40d->lli_log.src = NULL; 397 d40d->lli_log.dst = NULL; 398 d40d->lli_phy.src = NULL; 399 d40d->lli_phy.dst = NULL; 400 } 401 402 static int d40_lcla_alloc_one(struct d40_chan *d40c, 403 struct d40_desc *d40d) 404 { 405 unsigned long flags; 406 int i; 407 int ret = -EINVAL; 408 int p; 409 410 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); 411 412 p = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP; 413 414 /* 415 * Allocate both src and dst at the same time, therefore the half 416 * start on 1 since 0 can't be used since zero is used as end marker. 417 */ 418 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) { 419 if (!d40c->base->lcla_pool.alloc_map[p + i]) { 420 d40c->base->lcla_pool.alloc_map[p + i] = d40d; 421 d40d->lcla_alloc++; 422 ret = i; 423 break; 424 } 425 } 426 427 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); 428 429 return ret; 430 } 431 432 static int d40_lcla_free_all(struct d40_chan *d40c, 433 struct d40_desc *d40d) 434 { 435 unsigned long flags; 436 int i; 437 int ret = -EINVAL; 438 439 if (chan_is_physical(d40c)) 440 return 0; 441 442 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); 443 444 for (i = 1 ; i < D40_LCLA_LINK_PER_EVENT_GRP / 2; i++) { 445 if (d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num * 446 D40_LCLA_LINK_PER_EVENT_GRP + i] == d40d) { 447 d40c->base->lcla_pool.alloc_map[d40c->phy_chan->num * 448 D40_LCLA_LINK_PER_EVENT_GRP + i] = NULL; 449 d40d->lcla_alloc--; 450 if (d40d->lcla_alloc == 0) { 451 ret = 0; 452 break; 453 } 454 } 455 } 456 457 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); 458 459 return ret; 460 461 } 462 463 static void d40_desc_remove(struct d40_desc *d40d) 464 { 465 list_del(&d40d->node); 466 } 467 468 static struct d40_desc *d40_desc_get(struct d40_chan *d40c) 469 { 470 struct d40_desc *desc = NULL; 471 472 if (!list_empty(&d40c->client)) { 473 struct d40_desc *d; 474 struct d40_desc *_d; 475 476 list_for_each_entry_safe(d, _d, &d40c->client, node) 477 if (async_tx_test_ack(&d->txd)) { 478 d40_pool_lli_free(d40c, d); 479 d40_desc_remove(d); 480 desc = d; 481 memset(desc, 0, sizeof(*desc)); 482 break; 483 } 484 } 485 486 if (!desc) 487 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT); 488 489 if (desc) 490 INIT_LIST_HEAD(&desc->node); 491 492 return desc; 493 } 494 495 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) 496 { 497 498 d40_pool_lli_free(d40c, d40d); 499 d40_lcla_free_all(d40c, d40d); 500 kmem_cache_free(d40c->base->desc_slab, d40d); 501 } 502 503 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc) 504 { 505 list_add_tail(&desc->node, &d40c->active); 506 } 507 508 static void d40_phy_lli_load(struct d40_chan *chan, struct d40_desc *desc) 509 { 510 struct d40_phy_lli *lli_dst = desc->lli_phy.dst; 511 struct d40_phy_lli *lli_src = desc->lli_phy.src; 512 void __iomem *base = chan_base(chan); 513 514 writel(lli_src->reg_cfg, base + D40_CHAN_REG_SSCFG); 515 writel(lli_src->reg_elt, base + D40_CHAN_REG_SSELT); 516 writel(lli_src->reg_ptr, base + D40_CHAN_REG_SSPTR); 517 writel(lli_src->reg_lnk, base + D40_CHAN_REG_SSLNK); 518 519 writel(lli_dst->reg_cfg, base + D40_CHAN_REG_SDCFG); 520 writel(lli_dst->reg_elt, base + D40_CHAN_REG_SDELT); 521 writel(lli_dst->reg_ptr, base + D40_CHAN_REG_SDPTR); 522 writel(lli_dst->reg_lnk, base + D40_CHAN_REG_SDLNK); 523 } 524 525 static void d40_log_lli_to_lcxa(struct d40_chan *chan, struct d40_desc *desc) 526 { 527 struct d40_lcla_pool *pool = &chan->base->lcla_pool; 528 struct d40_log_lli_bidir *lli = &desc->lli_log; 529 int lli_current = desc->lli_current; 530 int lli_len = desc->lli_len; 531 bool cyclic = desc->cyclic; 532 int curr_lcla = -EINVAL; 533 int first_lcla = 0; 534 bool linkback; 535 536 /* 537 * We may have partially running cyclic transfers, in case we did't get 538 * enough LCLA entries. 539 */ 540 linkback = cyclic && lli_current == 0; 541 542 /* 543 * For linkback, we need one LCLA even with only one link, because we 544 * can't link back to the one in LCPA space 545 */ 546 if (linkback || (lli_len - lli_current > 1)) { 547 curr_lcla = d40_lcla_alloc_one(chan, desc); 548 first_lcla = curr_lcla; 549 } 550 551 /* 552 * For linkback, we normally load the LCPA in the loop since we need to 553 * link it to the second LCLA and not the first. However, if we 554 * couldn't even get a first LCLA, then we have to run in LCPA and 555 * reload manually. 556 */ 557 if (!linkback || curr_lcla == -EINVAL) { 558 unsigned int flags = 0; 559 560 if (curr_lcla == -EINVAL) 561 flags |= LLI_TERM_INT; 562 563 d40_log_lli_lcpa_write(chan->lcpa, 564 &lli->dst[lli_current], 565 &lli->src[lli_current], 566 curr_lcla, 567 flags); 568 lli_current++; 569 } 570 571 if (curr_lcla < 0) 572 goto out; 573 574 for (; lli_current < lli_len; lli_current++) { 575 unsigned int lcla_offset = chan->phy_chan->num * 1024 + 576 8 * curr_lcla * 2; 577 struct d40_log_lli *lcla = pool->base + lcla_offset; 578 unsigned int flags = 0; 579 int next_lcla; 580 581 if (lli_current + 1 < lli_len) 582 next_lcla = d40_lcla_alloc_one(chan, desc); 583 else 584 next_lcla = linkback ? first_lcla : -EINVAL; 585 586 if (cyclic || next_lcla == -EINVAL) 587 flags |= LLI_TERM_INT; 588 589 if (linkback && curr_lcla == first_lcla) { 590 /* First link goes in both LCPA and LCLA */ 591 d40_log_lli_lcpa_write(chan->lcpa, 592 &lli->dst[lli_current], 593 &lli->src[lli_current], 594 next_lcla, flags); 595 } 596 597 /* 598 * One unused LCLA in the cyclic case if the very first 599 * next_lcla fails... 600 */ 601 d40_log_lli_lcla_write(lcla, 602 &lli->dst[lli_current], 603 &lli->src[lli_current], 604 next_lcla, flags); 605 606 dma_sync_single_range_for_device(chan->base->dev, 607 pool->dma_addr, lcla_offset, 608 2 * sizeof(struct d40_log_lli), 609 DMA_TO_DEVICE); 610 611 curr_lcla = next_lcla; 612 613 if (curr_lcla == -EINVAL || curr_lcla == first_lcla) { 614 lli_current++; 615 break; 616 } 617 } 618 619 out: 620 desc->lli_current = lli_current; 621 } 622 623 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) 624 { 625 if (chan_is_physical(d40c)) { 626 d40_phy_lli_load(d40c, d40d); 627 d40d->lli_current = d40d->lli_len; 628 } else 629 d40_log_lli_to_lcxa(d40c, d40d); 630 } 631 632 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c) 633 { 634 struct d40_desc *d; 635 636 if (list_empty(&d40c->active)) 637 return NULL; 638 639 d = list_first_entry(&d40c->active, 640 struct d40_desc, 641 node); 642 return d; 643 } 644 645 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc) 646 { 647 list_add_tail(&desc->node, &d40c->queue); 648 } 649 650 static struct d40_desc *d40_first_queued(struct d40_chan *d40c) 651 { 652 struct d40_desc *d; 653 654 if (list_empty(&d40c->queue)) 655 return NULL; 656 657 d = list_first_entry(&d40c->queue, 658 struct d40_desc, 659 node); 660 return d; 661 } 662 663 static int d40_psize_2_burst_size(bool is_log, int psize) 664 { 665 if (is_log) { 666 if (psize == STEDMA40_PSIZE_LOG_1) 667 return 1; 668 } else { 669 if (psize == STEDMA40_PSIZE_PHY_1) 670 return 1; 671 } 672 673 return 2 << psize; 674 } 675 676 /* 677 * The dma only supports transmitting packages up to 678 * STEDMA40_MAX_SEG_SIZE << data_width. Calculate the total number of 679 * dma elements required to send the entire sg list 680 */ 681 static int d40_size_2_dmalen(int size, u32 data_width1, u32 data_width2) 682 { 683 int dmalen; 684 u32 max_w = max(data_width1, data_width2); 685 u32 min_w = min(data_width1, data_width2); 686 u32 seg_max = ALIGN(STEDMA40_MAX_SEG_SIZE << min_w, 1 << max_w); 687 688 if (seg_max > STEDMA40_MAX_SEG_SIZE) 689 seg_max -= (1 << max_w); 690 691 if (!IS_ALIGNED(size, 1 << max_w)) 692 return -EINVAL; 693 694 if (size <= seg_max) 695 dmalen = 1; 696 else { 697 dmalen = size / seg_max; 698 if (dmalen * seg_max < size) 699 dmalen++; 700 } 701 return dmalen; 702 } 703 704 static int d40_sg_2_dmalen(struct scatterlist *sgl, int sg_len, 705 u32 data_width1, u32 data_width2) 706 { 707 struct scatterlist *sg; 708 int i; 709 int len = 0; 710 int ret; 711 712 for_each_sg(sgl, sg, sg_len, i) { 713 ret = d40_size_2_dmalen(sg_dma_len(sg), 714 data_width1, data_width2); 715 if (ret < 0) 716 return ret; 717 len += ret; 718 } 719 return len; 720 } 721 722 /* Support functions for logical channels */ 723 724 static int d40_channel_execute_command(struct d40_chan *d40c, 725 enum d40_command command) 726 { 727 u32 status; 728 int i; 729 void __iomem *active_reg; 730 int ret = 0; 731 unsigned long flags; 732 u32 wmask; 733 734 spin_lock_irqsave(&d40c->base->execmd_lock, flags); 735 736 if (d40c->phy_chan->num % 2 == 0) 737 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; 738 else 739 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; 740 741 if (command == D40_DMA_SUSPEND_REQ) { 742 status = (readl(active_reg) & 743 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> 744 D40_CHAN_POS(d40c->phy_chan->num); 745 746 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) 747 goto done; 748 } 749 750 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num)); 751 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)), 752 active_reg); 753 754 if (command == D40_DMA_SUSPEND_REQ) { 755 756 for (i = 0 ; i < D40_SUSPEND_MAX_IT; i++) { 757 status = (readl(active_reg) & 758 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> 759 D40_CHAN_POS(d40c->phy_chan->num); 760 761 cpu_relax(); 762 /* 763 * Reduce the number of bus accesses while 764 * waiting for the DMA to suspend. 765 */ 766 udelay(3); 767 768 if (status == D40_DMA_STOP || 769 status == D40_DMA_SUSPENDED) 770 break; 771 } 772 773 if (i == D40_SUSPEND_MAX_IT) { 774 chan_err(d40c, 775 "unable to suspend the chl %d (log: %d) status %x\n", 776 d40c->phy_chan->num, d40c->log_num, 777 status); 778 dump_stack(); 779 ret = -EBUSY; 780 } 781 782 } 783 done: 784 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags); 785 return ret; 786 } 787 788 static void d40_term_all(struct d40_chan *d40c) 789 { 790 struct d40_desc *d40d; 791 792 /* Release active descriptors */ 793 while ((d40d = d40_first_active_get(d40c))) { 794 d40_desc_remove(d40d); 795 d40_desc_free(d40c, d40d); 796 } 797 798 /* Release queued descriptors waiting for transfer */ 799 while ((d40d = d40_first_queued(d40c))) { 800 d40_desc_remove(d40d); 801 d40_desc_free(d40c, d40d); 802 } 803 804 805 d40c->pending_tx = 0; 806 d40c->busy = false; 807 } 808 809 static void __d40_config_set_event(struct d40_chan *d40c, bool enable, 810 u32 event, int reg) 811 { 812 void __iomem *addr = chan_base(d40c) + reg; 813 int tries; 814 815 if (!enable) { 816 writel((D40_DEACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) 817 | ~D40_EVENTLINE_MASK(event), addr); 818 return; 819 } 820 821 /* 822 * The hardware sometimes doesn't register the enable when src and dst 823 * event lines are active on the same logical channel. Retry to ensure 824 * it does. Usually only one retry is sufficient. 825 */ 826 tries = 100; 827 while (--tries) { 828 writel((D40_ACTIVATE_EVENTLINE << D40_EVENTLINE_POS(event)) 829 | ~D40_EVENTLINE_MASK(event), addr); 830 831 if (readl(addr) & D40_EVENTLINE_MASK(event)) 832 break; 833 } 834 835 if (tries != 99) 836 dev_dbg(chan2dev(d40c), 837 "[%s] workaround enable S%cLNK (%d tries)\n", 838 __func__, reg == D40_CHAN_REG_SSLNK ? 'S' : 'D', 839 100 - tries); 840 841 WARN_ON(!tries); 842 } 843 844 static void d40_config_set_event(struct d40_chan *d40c, bool do_enable) 845 { 846 unsigned long flags; 847 848 spin_lock_irqsave(&d40c->phy_chan->lock, flags); 849 850 /* Enable event line connected to device (or memcpy) */ 851 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || 852 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) { 853 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 854 855 __d40_config_set_event(d40c, do_enable, event, 856 D40_CHAN_REG_SSLNK); 857 } 858 859 if (d40c->dma_cfg.dir != STEDMA40_PERIPH_TO_MEM) { 860 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 861 862 __d40_config_set_event(d40c, do_enable, event, 863 D40_CHAN_REG_SDLNK); 864 } 865 866 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags); 867 } 868 869 static u32 d40_chan_has_events(struct d40_chan *d40c) 870 { 871 void __iomem *chanbase = chan_base(d40c); 872 u32 val; 873 874 val = readl(chanbase + D40_CHAN_REG_SSLNK); 875 val |= readl(chanbase + D40_CHAN_REG_SDLNK); 876 877 return val; 878 } 879 880 static u32 d40_get_prmo(struct d40_chan *d40c) 881 { 882 static const unsigned int phy_map[] = { 883 [STEDMA40_PCHAN_BASIC_MODE] 884 = D40_DREG_PRMO_PCHAN_BASIC, 885 [STEDMA40_PCHAN_MODULO_MODE] 886 = D40_DREG_PRMO_PCHAN_MODULO, 887 [STEDMA40_PCHAN_DOUBLE_DST_MODE] 888 = D40_DREG_PRMO_PCHAN_DOUBLE_DST, 889 }; 890 static const unsigned int log_map[] = { 891 [STEDMA40_LCHAN_SRC_PHY_DST_LOG] 892 = D40_DREG_PRMO_LCHAN_SRC_PHY_DST_LOG, 893 [STEDMA40_LCHAN_SRC_LOG_DST_PHY] 894 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_PHY, 895 [STEDMA40_LCHAN_SRC_LOG_DST_LOG] 896 = D40_DREG_PRMO_LCHAN_SRC_LOG_DST_LOG, 897 }; 898 899 if (chan_is_physical(d40c)) 900 return phy_map[d40c->dma_cfg.mode_opt]; 901 else 902 return log_map[d40c->dma_cfg.mode_opt]; 903 } 904 905 static void d40_config_write(struct d40_chan *d40c) 906 { 907 u32 addr_base; 908 u32 var; 909 910 /* Odd addresses are even addresses + 4 */ 911 addr_base = (d40c->phy_chan->num % 2) * 4; 912 /* Setup channel mode to logical or physical */ 913 var = ((u32)(chan_is_logical(d40c)) + 1) << 914 D40_CHAN_POS(d40c->phy_chan->num); 915 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base); 916 917 /* Setup operational mode option register */ 918 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num); 919 920 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base); 921 922 if (chan_is_logical(d40c)) { 923 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) 924 & D40_SREG_ELEM_LOG_LIDX_MASK; 925 void __iomem *chanbase = chan_base(d40c); 926 927 /* Set default config for CFG reg */ 928 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG); 929 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG); 930 931 /* Set LIDX for lcla */ 932 writel(lidx, chanbase + D40_CHAN_REG_SSELT); 933 writel(lidx, chanbase + D40_CHAN_REG_SDELT); 934 } 935 } 936 937 static u32 d40_residue(struct d40_chan *d40c) 938 { 939 u32 num_elt; 940 941 if (chan_is_logical(d40c)) 942 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) 943 >> D40_MEM_LCSP2_ECNT_POS; 944 else { 945 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT); 946 num_elt = (val & D40_SREG_ELEM_PHY_ECNT_MASK) 947 >> D40_SREG_ELEM_PHY_ECNT_POS; 948 } 949 950 return num_elt * (1 << d40c->dma_cfg.dst_info.data_width); 951 } 952 953 static bool d40_tx_is_linked(struct d40_chan *d40c) 954 { 955 bool is_link; 956 957 if (chan_is_logical(d40c)) 958 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK; 959 else 960 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK) 961 & D40_SREG_LNK_PHYS_LNK_MASK; 962 963 return is_link; 964 } 965 966 static int d40_pause(struct d40_chan *d40c) 967 { 968 int res = 0; 969 unsigned long flags; 970 971 if (!d40c->busy) 972 return 0; 973 974 spin_lock_irqsave(&d40c->lock, flags); 975 976 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 977 if (res == 0) { 978 if (chan_is_logical(d40c)) { 979 d40_config_set_event(d40c, false); 980 /* Resume the other logical channels if any */ 981 if (d40_chan_has_events(d40c)) 982 res = d40_channel_execute_command(d40c, 983 D40_DMA_RUN); 984 } 985 } 986 987 spin_unlock_irqrestore(&d40c->lock, flags); 988 return res; 989 } 990 991 static int d40_resume(struct d40_chan *d40c) 992 { 993 int res = 0; 994 unsigned long flags; 995 996 if (!d40c->busy) 997 return 0; 998 999 spin_lock_irqsave(&d40c->lock, flags); 1000 1001 if (d40c->base->rev == 0) 1002 if (chan_is_logical(d40c)) { 1003 res = d40_channel_execute_command(d40c, 1004 D40_DMA_SUSPEND_REQ); 1005 goto no_suspend; 1006 } 1007 1008 /* If bytes left to transfer or linked tx resume job */ 1009 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) { 1010 1011 if (chan_is_logical(d40c)) 1012 d40_config_set_event(d40c, true); 1013 1014 res = d40_channel_execute_command(d40c, D40_DMA_RUN); 1015 } 1016 1017 no_suspend: 1018 spin_unlock_irqrestore(&d40c->lock, flags); 1019 return res; 1020 } 1021 1022 static int d40_terminate_all(struct d40_chan *chan) 1023 { 1024 unsigned long flags; 1025 int ret = 0; 1026 1027 ret = d40_pause(chan); 1028 if (!ret && chan_is_physical(chan)) 1029 ret = d40_channel_execute_command(chan, D40_DMA_STOP); 1030 1031 spin_lock_irqsave(&chan->lock, flags); 1032 d40_term_all(chan); 1033 spin_unlock_irqrestore(&chan->lock, flags); 1034 1035 return ret; 1036 } 1037 1038 static dma_cookie_t d40_tx_submit(struct dma_async_tx_descriptor *tx) 1039 { 1040 struct d40_chan *d40c = container_of(tx->chan, 1041 struct d40_chan, 1042 chan); 1043 struct d40_desc *d40d = container_of(tx, struct d40_desc, txd); 1044 unsigned long flags; 1045 1046 spin_lock_irqsave(&d40c->lock, flags); 1047 1048 d40c->chan.cookie++; 1049 1050 if (d40c->chan.cookie < 0) 1051 d40c->chan.cookie = 1; 1052 1053 d40d->txd.cookie = d40c->chan.cookie; 1054 1055 d40_desc_queue(d40c, d40d); 1056 1057 spin_unlock_irqrestore(&d40c->lock, flags); 1058 1059 return tx->cookie; 1060 } 1061 1062 static int d40_start(struct d40_chan *d40c) 1063 { 1064 if (d40c->base->rev == 0) { 1065 int err; 1066 1067 if (chan_is_logical(d40c)) { 1068 err = d40_channel_execute_command(d40c, 1069 D40_DMA_SUSPEND_REQ); 1070 if (err) 1071 return err; 1072 } 1073 } 1074 1075 if (chan_is_logical(d40c)) 1076 d40_config_set_event(d40c, true); 1077 1078 return d40_channel_execute_command(d40c, D40_DMA_RUN); 1079 } 1080 1081 static struct d40_desc *d40_queue_start(struct d40_chan *d40c) 1082 { 1083 struct d40_desc *d40d; 1084 int err; 1085 1086 /* Start queued jobs, if any */ 1087 d40d = d40_first_queued(d40c); 1088 1089 if (d40d != NULL) { 1090 d40c->busy = true; 1091 1092 /* Remove from queue */ 1093 d40_desc_remove(d40d); 1094 1095 /* Add to active queue */ 1096 d40_desc_submit(d40c, d40d); 1097 1098 /* Initiate DMA job */ 1099 d40_desc_load(d40c, d40d); 1100 1101 /* Start dma job */ 1102 err = d40_start(d40c); 1103 1104 if (err) 1105 return NULL; 1106 } 1107 1108 return d40d; 1109 } 1110 1111 /* called from interrupt context */ 1112 static void dma_tc_handle(struct d40_chan *d40c) 1113 { 1114 struct d40_desc *d40d; 1115 1116 /* Get first active entry from list */ 1117 d40d = d40_first_active_get(d40c); 1118 1119 if (d40d == NULL) 1120 return; 1121 1122 if (d40d->cyclic) { 1123 /* 1124 * If this was a paritially loaded list, we need to reloaded 1125 * it, and only when the list is completed. We need to check 1126 * for done because the interrupt will hit for every link, and 1127 * not just the last one. 1128 */ 1129 if (d40d->lli_current < d40d->lli_len 1130 && !d40_tx_is_linked(d40c) 1131 && !d40_residue(d40c)) { 1132 d40_lcla_free_all(d40c, d40d); 1133 d40_desc_load(d40c, d40d); 1134 (void) d40_start(d40c); 1135 1136 if (d40d->lli_current == d40d->lli_len) 1137 d40d->lli_current = 0; 1138 } 1139 } else { 1140 d40_lcla_free_all(d40c, d40d); 1141 1142 if (d40d->lli_current < d40d->lli_len) { 1143 d40_desc_load(d40c, d40d); 1144 /* Start dma job */ 1145 (void) d40_start(d40c); 1146 return; 1147 } 1148 1149 if (d40_queue_start(d40c) == NULL) 1150 d40c->busy = false; 1151 } 1152 1153 d40c->pending_tx++; 1154 tasklet_schedule(&d40c->tasklet); 1155 1156 } 1157 1158 static void dma_tasklet(unsigned long data) 1159 { 1160 struct d40_chan *d40c = (struct d40_chan *) data; 1161 struct d40_desc *d40d; 1162 unsigned long flags; 1163 dma_async_tx_callback callback; 1164 void *callback_param; 1165 1166 spin_lock_irqsave(&d40c->lock, flags); 1167 1168 /* Get first active entry from list */ 1169 d40d = d40_first_active_get(d40c); 1170 if (d40d == NULL) 1171 goto err; 1172 1173 if (!d40d->cyclic) 1174 d40c->completed = d40d->txd.cookie; 1175 1176 /* 1177 * If terminating a channel pending_tx is set to zero. 1178 * This prevents any finished active jobs to return to the client. 1179 */ 1180 if (d40c->pending_tx == 0) { 1181 spin_unlock_irqrestore(&d40c->lock, flags); 1182 return; 1183 } 1184 1185 /* Callback to client */ 1186 callback = d40d->txd.callback; 1187 callback_param = d40d->txd.callback_param; 1188 1189 if (!d40d->cyclic) { 1190 if (async_tx_test_ack(&d40d->txd)) { 1191 d40_pool_lli_free(d40c, d40d); 1192 d40_desc_remove(d40d); 1193 d40_desc_free(d40c, d40d); 1194 } else { 1195 if (!d40d->is_in_client_list) { 1196 d40_desc_remove(d40d); 1197 d40_lcla_free_all(d40c, d40d); 1198 list_add_tail(&d40d->node, &d40c->client); 1199 d40d->is_in_client_list = true; 1200 } 1201 } 1202 } 1203 1204 d40c->pending_tx--; 1205 1206 if (d40c->pending_tx) 1207 tasklet_schedule(&d40c->tasklet); 1208 1209 spin_unlock_irqrestore(&d40c->lock, flags); 1210 1211 if (callback && (d40d->txd.flags & DMA_PREP_INTERRUPT)) 1212 callback(callback_param); 1213 1214 return; 1215 1216 err: 1217 /* Rescue manoeuvre if receiving double interrupts */ 1218 if (d40c->pending_tx > 0) 1219 d40c->pending_tx--; 1220 spin_unlock_irqrestore(&d40c->lock, flags); 1221 } 1222 1223 static irqreturn_t d40_handle_interrupt(int irq, void *data) 1224 { 1225 static const struct d40_interrupt_lookup il[] = { 1226 {D40_DREG_LCTIS0, D40_DREG_LCICR0, false, 0}, 1227 {D40_DREG_LCTIS1, D40_DREG_LCICR1, false, 32}, 1228 {D40_DREG_LCTIS2, D40_DREG_LCICR2, false, 64}, 1229 {D40_DREG_LCTIS3, D40_DREG_LCICR3, false, 96}, 1230 {D40_DREG_LCEIS0, D40_DREG_LCICR0, true, 0}, 1231 {D40_DREG_LCEIS1, D40_DREG_LCICR1, true, 32}, 1232 {D40_DREG_LCEIS2, D40_DREG_LCICR2, true, 64}, 1233 {D40_DREG_LCEIS3, D40_DREG_LCICR3, true, 96}, 1234 {D40_DREG_PCTIS, D40_DREG_PCICR, false, D40_PHY_CHAN}, 1235 {D40_DREG_PCEIS, D40_DREG_PCICR, true, D40_PHY_CHAN}, 1236 }; 1237 1238 int i; 1239 u32 regs[ARRAY_SIZE(il)]; 1240 u32 idx; 1241 u32 row; 1242 long chan = -1; 1243 struct d40_chan *d40c; 1244 unsigned long flags; 1245 struct d40_base *base = data; 1246 1247 spin_lock_irqsave(&base->interrupt_lock, flags); 1248 1249 /* Read interrupt status of both logical and physical channels */ 1250 for (i = 0; i < ARRAY_SIZE(il); i++) 1251 regs[i] = readl(base->virtbase + il[i].src); 1252 1253 for (;;) { 1254 1255 chan = find_next_bit((unsigned long *)regs, 1256 BITS_PER_LONG * ARRAY_SIZE(il), chan + 1); 1257 1258 /* No more set bits found? */ 1259 if (chan == BITS_PER_LONG * ARRAY_SIZE(il)) 1260 break; 1261 1262 row = chan / BITS_PER_LONG; 1263 idx = chan & (BITS_PER_LONG - 1); 1264 1265 /* ACK interrupt */ 1266 writel(1 << idx, base->virtbase + il[row].clr); 1267 1268 if (il[row].offset == D40_PHY_CHAN) 1269 d40c = base->lookup_phy_chans[idx]; 1270 else 1271 d40c = base->lookup_log_chans[il[row].offset + idx]; 1272 spin_lock(&d40c->lock); 1273 1274 if (!il[row].is_error) 1275 dma_tc_handle(d40c); 1276 else 1277 d40_err(base->dev, "IRQ chan: %ld offset %d idx %d\n", 1278 chan, il[row].offset, idx); 1279 1280 spin_unlock(&d40c->lock); 1281 } 1282 1283 spin_unlock_irqrestore(&base->interrupt_lock, flags); 1284 1285 return IRQ_HANDLED; 1286 } 1287 1288 static int d40_validate_conf(struct d40_chan *d40c, 1289 struct stedma40_chan_cfg *conf) 1290 { 1291 int res = 0; 1292 u32 dst_event_group = D40_TYPE_TO_GROUP(conf->dst_dev_type); 1293 u32 src_event_group = D40_TYPE_TO_GROUP(conf->src_dev_type); 1294 bool is_log = conf->mode == STEDMA40_MODE_LOGICAL; 1295 1296 if (!conf->dir) { 1297 chan_err(d40c, "Invalid direction.\n"); 1298 res = -EINVAL; 1299 } 1300 1301 if (conf->dst_dev_type != STEDMA40_DEV_DST_MEMORY && 1302 d40c->base->plat_data->dev_tx[conf->dst_dev_type] == 0 && 1303 d40c->runtime_addr == 0) { 1304 1305 chan_err(d40c, "Invalid TX channel address (%d)\n", 1306 conf->dst_dev_type); 1307 res = -EINVAL; 1308 } 1309 1310 if (conf->src_dev_type != STEDMA40_DEV_SRC_MEMORY && 1311 d40c->base->plat_data->dev_rx[conf->src_dev_type] == 0 && 1312 d40c->runtime_addr == 0) { 1313 chan_err(d40c, "Invalid RX channel address (%d)\n", 1314 conf->src_dev_type); 1315 res = -EINVAL; 1316 } 1317 1318 if (conf->dir == STEDMA40_MEM_TO_PERIPH && 1319 dst_event_group == STEDMA40_DEV_DST_MEMORY) { 1320 chan_err(d40c, "Invalid dst\n"); 1321 res = -EINVAL; 1322 } 1323 1324 if (conf->dir == STEDMA40_PERIPH_TO_MEM && 1325 src_event_group == STEDMA40_DEV_SRC_MEMORY) { 1326 chan_err(d40c, "Invalid src\n"); 1327 res = -EINVAL; 1328 } 1329 1330 if (src_event_group == STEDMA40_DEV_SRC_MEMORY && 1331 dst_event_group == STEDMA40_DEV_DST_MEMORY && is_log) { 1332 chan_err(d40c, "No event line\n"); 1333 res = -EINVAL; 1334 } 1335 1336 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH && 1337 (src_event_group != dst_event_group)) { 1338 chan_err(d40c, "Invalid event group\n"); 1339 res = -EINVAL; 1340 } 1341 1342 if (conf->dir == STEDMA40_PERIPH_TO_PERIPH) { 1343 /* 1344 * DMAC HW supports it. Will be added to this driver, 1345 * in case any dma client requires it. 1346 */ 1347 chan_err(d40c, "periph to periph not supported\n"); 1348 res = -EINVAL; 1349 } 1350 1351 if (d40_psize_2_burst_size(is_log, conf->src_info.psize) * 1352 (1 << conf->src_info.data_width) != 1353 d40_psize_2_burst_size(is_log, conf->dst_info.psize) * 1354 (1 << conf->dst_info.data_width)) { 1355 /* 1356 * The DMAC hardware only supports 1357 * src (burst x width) == dst (burst x width) 1358 */ 1359 1360 chan_err(d40c, "src (burst x width) != dst (burst x width)\n"); 1361 res = -EINVAL; 1362 } 1363 1364 return res; 1365 } 1366 1367 static bool d40_alloc_mask_set(struct d40_phy_res *phy, bool is_src, 1368 int log_event_line, bool is_log) 1369 { 1370 unsigned long flags; 1371 spin_lock_irqsave(&phy->lock, flags); 1372 if (!is_log) { 1373 /* Physical interrupts are masked per physical full channel */ 1374 if (phy->allocated_src == D40_ALLOC_FREE && 1375 phy->allocated_dst == D40_ALLOC_FREE) { 1376 phy->allocated_dst = D40_ALLOC_PHY; 1377 phy->allocated_src = D40_ALLOC_PHY; 1378 goto found; 1379 } else 1380 goto not_found; 1381 } 1382 1383 /* Logical channel */ 1384 if (is_src) { 1385 if (phy->allocated_src == D40_ALLOC_PHY) 1386 goto not_found; 1387 1388 if (phy->allocated_src == D40_ALLOC_FREE) 1389 phy->allocated_src = D40_ALLOC_LOG_FREE; 1390 1391 if (!(phy->allocated_src & (1 << log_event_line))) { 1392 phy->allocated_src |= 1 << log_event_line; 1393 goto found; 1394 } else 1395 goto not_found; 1396 } else { 1397 if (phy->allocated_dst == D40_ALLOC_PHY) 1398 goto not_found; 1399 1400 if (phy->allocated_dst == D40_ALLOC_FREE) 1401 phy->allocated_dst = D40_ALLOC_LOG_FREE; 1402 1403 if (!(phy->allocated_dst & (1 << log_event_line))) { 1404 phy->allocated_dst |= 1 << log_event_line; 1405 goto found; 1406 } else 1407 goto not_found; 1408 } 1409 1410 not_found: 1411 spin_unlock_irqrestore(&phy->lock, flags); 1412 return false; 1413 found: 1414 spin_unlock_irqrestore(&phy->lock, flags); 1415 return true; 1416 } 1417 1418 static bool d40_alloc_mask_free(struct d40_phy_res *phy, bool is_src, 1419 int log_event_line) 1420 { 1421 unsigned long flags; 1422 bool is_free = false; 1423 1424 spin_lock_irqsave(&phy->lock, flags); 1425 if (!log_event_line) { 1426 phy->allocated_dst = D40_ALLOC_FREE; 1427 phy->allocated_src = D40_ALLOC_FREE; 1428 is_free = true; 1429 goto out; 1430 } 1431 1432 /* Logical channel */ 1433 if (is_src) { 1434 phy->allocated_src &= ~(1 << log_event_line); 1435 if (phy->allocated_src == D40_ALLOC_LOG_FREE) 1436 phy->allocated_src = D40_ALLOC_FREE; 1437 } else { 1438 phy->allocated_dst &= ~(1 << log_event_line); 1439 if (phy->allocated_dst == D40_ALLOC_LOG_FREE) 1440 phy->allocated_dst = D40_ALLOC_FREE; 1441 } 1442 1443 is_free = ((phy->allocated_src | phy->allocated_dst) == 1444 D40_ALLOC_FREE); 1445 1446 out: 1447 spin_unlock_irqrestore(&phy->lock, flags); 1448 1449 return is_free; 1450 } 1451 1452 static int d40_allocate_channel(struct d40_chan *d40c) 1453 { 1454 int dev_type; 1455 int event_group; 1456 int event_line; 1457 struct d40_phy_res *phys; 1458 int i; 1459 int j; 1460 int log_num; 1461 bool is_src; 1462 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL; 1463 1464 phys = d40c->base->phy_res; 1465 1466 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1467 dev_type = d40c->dma_cfg.src_dev_type; 1468 log_num = 2 * dev_type; 1469 is_src = true; 1470 } else if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1471 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1472 /* dst event lines are used for logical memcpy */ 1473 dev_type = d40c->dma_cfg.dst_dev_type; 1474 log_num = 2 * dev_type + 1; 1475 is_src = false; 1476 } else 1477 return -EINVAL; 1478 1479 event_group = D40_TYPE_TO_GROUP(dev_type); 1480 event_line = D40_TYPE_TO_EVENT(dev_type); 1481 1482 if (!is_log) { 1483 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1484 /* Find physical half channel */ 1485 for (i = 0; i < d40c->base->num_phy_chans; i++) { 1486 1487 if (d40_alloc_mask_set(&phys[i], is_src, 1488 0, is_log)) 1489 goto found_phy; 1490 } 1491 } else 1492 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { 1493 int phy_num = j + event_group * 2; 1494 for (i = phy_num; i < phy_num + 2; i++) { 1495 if (d40_alloc_mask_set(&phys[i], 1496 is_src, 1497 0, 1498 is_log)) 1499 goto found_phy; 1500 } 1501 } 1502 return -EINVAL; 1503 found_phy: 1504 d40c->phy_chan = &phys[i]; 1505 d40c->log_num = D40_PHY_CHAN; 1506 goto out; 1507 } 1508 if (dev_type == -1) 1509 return -EINVAL; 1510 1511 /* Find logical channel */ 1512 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { 1513 int phy_num = j + event_group * 2; 1514 /* 1515 * Spread logical channels across all available physical rather 1516 * than pack every logical channel at the first available phy 1517 * channels. 1518 */ 1519 if (is_src) { 1520 for (i = phy_num; i < phy_num + 2; i++) { 1521 if (d40_alloc_mask_set(&phys[i], is_src, 1522 event_line, is_log)) 1523 goto found_log; 1524 } 1525 } else { 1526 for (i = phy_num + 1; i >= phy_num; i--) { 1527 if (d40_alloc_mask_set(&phys[i], is_src, 1528 event_line, is_log)) 1529 goto found_log; 1530 } 1531 } 1532 } 1533 return -EINVAL; 1534 1535 found_log: 1536 d40c->phy_chan = &phys[i]; 1537 d40c->log_num = log_num; 1538 out: 1539 1540 if (is_log) 1541 d40c->base->lookup_log_chans[d40c->log_num] = d40c; 1542 else 1543 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c; 1544 1545 return 0; 1546 1547 } 1548 1549 static int d40_config_memcpy(struct d40_chan *d40c) 1550 { 1551 dma_cap_mask_t cap = d40c->chan.device->cap_mask; 1552 1553 if (dma_has_cap(DMA_MEMCPY, cap) && !dma_has_cap(DMA_SLAVE, cap)) { 1554 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_log; 1555 d40c->dma_cfg.src_dev_type = STEDMA40_DEV_SRC_MEMORY; 1556 d40c->dma_cfg.dst_dev_type = d40c->base->plat_data-> 1557 memcpy[d40c->chan.chan_id]; 1558 1559 } else if (dma_has_cap(DMA_MEMCPY, cap) && 1560 dma_has_cap(DMA_SLAVE, cap)) { 1561 d40c->dma_cfg = *d40c->base->plat_data->memcpy_conf_phy; 1562 } else { 1563 chan_err(d40c, "No memcpy\n"); 1564 return -EINVAL; 1565 } 1566 1567 return 0; 1568 } 1569 1570 1571 static int d40_free_dma(struct d40_chan *d40c) 1572 { 1573 1574 int res = 0; 1575 u32 event; 1576 struct d40_phy_res *phy = d40c->phy_chan; 1577 bool is_src; 1578 struct d40_desc *d; 1579 struct d40_desc *_d; 1580 1581 1582 /* Terminate all queued and active transfers */ 1583 d40_term_all(d40c); 1584 1585 /* Release client owned descriptors */ 1586 if (!list_empty(&d40c->client)) 1587 list_for_each_entry_safe(d, _d, &d40c->client, node) { 1588 d40_pool_lli_free(d40c, d); 1589 d40_desc_remove(d); 1590 d40_desc_free(d40c, d); 1591 } 1592 1593 if (phy == NULL) { 1594 chan_err(d40c, "phy == null\n"); 1595 return -EINVAL; 1596 } 1597 1598 if (phy->allocated_src == D40_ALLOC_FREE && 1599 phy->allocated_dst == D40_ALLOC_FREE) { 1600 chan_err(d40c, "channel already free\n"); 1601 return -EINVAL; 1602 } 1603 1604 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1605 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1606 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1607 is_src = false; 1608 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1609 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1610 is_src = true; 1611 } else { 1612 chan_err(d40c, "Unknown direction\n"); 1613 return -EINVAL; 1614 } 1615 1616 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); 1617 if (res) { 1618 chan_err(d40c, "suspend failed\n"); 1619 return res; 1620 } 1621 1622 if (chan_is_logical(d40c)) { 1623 /* Release logical channel, deactivate the event line */ 1624 1625 d40_config_set_event(d40c, false); 1626 d40c->base->lookup_log_chans[d40c->log_num] = NULL; 1627 1628 /* 1629 * Check if there are more logical allocation 1630 * on this phy channel. 1631 */ 1632 if (!d40_alloc_mask_free(phy, is_src, event)) { 1633 /* Resume the other logical channels if any */ 1634 if (d40_chan_has_events(d40c)) { 1635 res = d40_channel_execute_command(d40c, 1636 D40_DMA_RUN); 1637 if (res) { 1638 chan_err(d40c, 1639 "Executing RUN command\n"); 1640 return res; 1641 } 1642 } 1643 return 0; 1644 } 1645 } else { 1646 (void) d40_alloc_mask_free(phy, is_src, 0); 1647 } 1648 1649 /* Release physical channel */ 1650 res = d40_channel_execute_command(d40c, D40_DMA_STOP); 1651 if (res) { 1652 chan_err(d40c, "Failed to stop channel\n"); 1653 return res; 1654 } 1655 d40c->phy_chan = NULL; 1656 d40c->configured = false; 1657 d40c->base->lookup_phy_chans[phy->num] = NULL; 1658 1659 return 0; 1660 } 1661 1662 static bool d40_is_paused(struct d40_chan *d40c) 1663 { 1664 void __iomem *chanbase = chan_base(d40c); 1665 bool is_paused = false; 1666 unsigned long flags; 1667 void __iomem *active_reg; 1668 u32 status; 1669 u32 event; 1670 1671 spin_lock_irqsave(&d40c->lock, flags); 1672 1673 if (chan_is_physical(d40c)) { 1674 if (d40c->phy_chan->num % 2 == 0) 1675 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; 1676 else 1677 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; 1678 1679 status = (readl(active_reg) & 1680 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> 1681 D40_CHAN_POS(d40c->phy_chan->num); 1682 if (status == D40_DMA_SUSPENDED || status == D40_DMA_STOP) 1683 is_paused = true; 1684 1685 goto _exit; 1686 } 1687 1688 if (d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH || 1689 d40c->dma_cfg.dir == STEDMA40_MEM_TO_MEM) { 1690 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dst_dev_type); 1691 status = readl(chanbase + D40_CHAN_REG_SDLNK); 1692 } else if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) { 1693 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.src_dev_type); 1694 status = readl(chanbase + D40_CHAN_REG_SSLNK); 1695 } else { 1696 chan_err(d40c, "Unknown direction\n"); 1697 goto _exit; 1698 } 1699 1700 status = (status & D40_EVENTLINE_MASK(event)) >> 1701 D40_EVENTLINE_POS(event); 1702 1703 if (status != D40_DMA_RUN) 1704 is_paused = true; 1705 _exit: 1706 spin_unlock_irqrestore(&d40c->lock, flags); 1707 return is_paused; 1708 1709 } 1710 1711 1712 static u32 stedma40_residue(struct dma_chan *chan) 1713 { 1714 struct d40_chan *d40c = 1715 container_of(chan, struct d40_chan, chan); 1716 u32 bytes_left; 1717 unsigned long flags; 1718 1719 spin_lock_irqsave(&d40c->lock, flags); 1720 bytes_left = d40_residue(d40c); 1721 spin_unlock_irqrestore(&d40c->lock, flags); 1722 1723 return bytes_left; 1724 } 1725 1726 static int 1727 d40_prep_sg_log(struct d40_chan *chan, struct d40_desc *desc, 1728 struct scatterlist *sg_src, struct scatterlist *sg_dst, 1729 unsigned int sg_len, dma_addr_t src_dev_addr, 1730 dma_addr_t dst_dev_addr) 1731 { 1732 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1733 struct stedma40_half_channel_info *src_info = &cfg->src_info; 1734 struct stedma40_half_channel_info *dst_info = &cfg->dst_info; 1735 int ret; 1736 1737 ret = d40_log_sg_to_lli(sg_src, sg_len, 1738 src_dev_addr, 1739 desc->lli_log.src, 1740 chan->log_def.lcsp1, 1741 src_info->data_width, 1742 dst_info->data_width); 1743 1744 ret = d40_log_sg_to_lli(sg_dst, sg_len, 1745 dst_dev_addr, 1746 desc->lli_log.dst, 1747 chan->log_def.lcsp3, 1748 dst_info->data_width, 1749 src_info->data_width); 1750 1751 return ret < 0 ? ret : 0; 1752 } 1753 1754 static int 1755 d40_prep_sg_phy(struct d40_chan *chan, struct d40_desc *desc, 1756 struct scatterlist *sg_src, struct scatterlist *sg_dst, 1757 unsigned int sg_len, dma_addr_t src_dev_addr, 1758 dma_addr_t dst_dev_addr) 1759 { 1760 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1761 struct stedma40_half_channel_info *src_info = &cfg->src_info; 1762 struct stedma40_half_channel_info *dst_info = &cfg->dst_info; 1763 unsigned long flags = 0; 1764 int ret; 1765 1766 if (desc->cyclic) 1767 flags |= LLI_CYCLIC | LLI_TERM_INT; 1768 1769 ret = d40_phy_sg_to_lli(sg_src, sg_len, src_dev_addr, 1770 desc->lli_phy.src, 1771 virt_to_phys(desc->lli_phy.src), 1772 chan->src_def_cfg, 1773 src_info, dst_info, flags); 1774 1775 ret = d40_phy_sg_to_lli(sg_dst, sg_len, dst_dev_addr, 1776 desc->lli_phy.dst, 1777 virt_to_phys(desc->lli_phy.dst), 1778 chan->dst_def_cfg, 1779 dst_info, src_info, flags); 1780 1781 dma_sync_single_for_device(chan->base->dev, desc->lli_pool.dma_addr, 1782 desc->lli_pool.size, DMA_TO_DEVICE); 1783 1784 return ret < 0 ? ret : 0; 1785 } 1786 1787 1788 static struct d40_desc * 1789 d40_prep_desc(struct d40_chan *chan, struct scatterlist *sg, 1790 unsigned int sg_len, unsigned long dma_flags) 1791 { 1792 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1793 struct d40_desc *desc; 1794 int ret; 1795 1796 desc = d40_desc_get(chan); 1797 if (!desc) 1798 return NULL; 1799 1800 desc->lli_len = d40_sg_2_dmalen(sg, sg_len, cfg->src_info.data_width, 1801 cfg->dst_info.data_width); 1802 if (desc->lli_len < 0) { 1803 chan_err(chan, "Unaligned size\n"); 1804 goto err; 1805 } 1806 1807 ret = d40_pool_lli_alloc(chan, desc, desc->lli_len); 1808 if (ret < 0) { 1809 chan_err(chan, "Could not allocate lli\n"); 1810 goto err; 1811 } 1812 1813 1814 desc->lli_current = 0; 1815 desc->txd.flags = dma_flags; 1816 desc->txd.tx_submit = d40_tx_submit; 1817 1818 dma_async_tx_descriptor_init(&desc->txd, &chan->chan); 1819 1820 return desc; 1821 1822 err: 1823 d40_desc_free(chan, desc); 1824 return NULL; 1825 } 1826 1827 static dma_addr_t 1828 d40_get_dev_addr(struct d40_chan *chan, enum dma_data_direction direction) 1829 { 1830 struct stedma40_platform_data *plat = chan->base->plat_data; 1831 struct stedma40_chan_cfg *cfg = &chan->dma_cfg; 1832 dma_addr_t addr = 0; 1833 1834 if (chan->runtime_addr) 1835 return chan->runtime_addr; 1836 1837 if (direction == DMA_FROM_DEVICE) 1838 addr = plat->dev_rx[cfg->src_dev_type]; 1839 else if (direction == DMA_TO_DEVICE) 1840 addr = plat->dev_tx[cfg->dst_dev_type]; 1841 1842 return addr; 1843 } 1844 1845 static struct dma_async_tx_descriptor * 1846 d40_prep_sg(struct dma_chan *dchan, struct scatterlist *sg_src, 1847 struct scatterlist *sg_dst, unsigned int sg_len, 1848 enum dma_data_direction direction, unsigned long dma_flags) 1849 { 1850 struct d40_chan *chan = container_of(dchan, struct d40_chan, chan); 1851 dma_addr_t src_dev_addr = 0; 1852 dma_addr_t dst_dev_addr = 0; 1853 struct d40_desc *desc; 1854 unsigned long flags; 1855 int ret; 1856 1857 if (!chan->phy_chan) { 1858 chan_err(chan, "Cannot prepare unallocated channel\n"); 1859 return NULL; 1860 } 1861 1862 1863 spin_lock_irqsave(&chan->lock, flags); 1864 1865 desc = d40_prep_desc(chan, sg_src, sg_len, dma_flags); 1866 if (desc == NULL) 1867 goto err; 1868 1869 if (sg_next(&sg_src[sg_len - 1]) == sg_src) 1870 desc->cyclic = true; 1871 1872 if (direction != DMA_NONE) { 1873 dma_addr_t dev_addr = d40_get_dev_addr(chan, direction); 1874 1875 if (direction == DMA_FROM_DEVICE) 1876 src_dev_addr = dev_addr; 1877 else if (direction == DMA_TO_DEVICE) 1878 dst_dev_addr = dev_addr; 1879 } 1880 1881 if (chan_is_logical(chan)) 1882 ret = d40_prep_sg_log(chan, desc, sg_src, sg_dst, 1883 sg_len, src_dev_addr, dst_dev_addr); 1884 else 1885 ret = d40_prep_sg_phy(chan, desc, sg_src, sg_dst, 1886 sg_len, src_dev_addr, dst_dev_addr); 1887 1888 if (ret) { 1889 chan_err(chan, "Failed to prepare %s sg job: %d\n", 1890 chan_is_logical(chan) ? "log" : "phy", ret); 1891 goto err; 1892 } 1893 1894 spin_unlock_irqrestore(&chan->lock, flags); 1895 1896 return &desc->txd; 1897 1898 err: 1899 if (desc) 1900 d40_desc_free(chan, desc); 1901 spin_unlock_irqrestore(&chan->lock, flags); 1902 return NULL; 1903 } 1904 1905 bool stedma40_filter(struct dma_chan *chan, void *data) 1906 { 1907 struct stedma40_chan_cfg *info = data; 1908 struct d40_chan *d40c = 1909 container_of(chan, struct d40_chan, chan); 1910 int err; 1911 1912 if (data) { 1913 err = d40_validate_conf(d40c, info); 1914 if (!err) 1915 d40c->dma_cfg = *info; 1916 } else 1917 err = d40_config_memcpy(d40c); 1918 1919 if (!err) 1920 d40c->configured = true; 1921 1922 return err == 0; 1923 } 1924 EXPORT_SYMBOL(stedma40_filter); 1925 1926 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src) 1927 { 1928 bool realtime = d40c->dma_cfg.realtime; 1929 bool highprio = d40c->dma_cfg.high_priority; 1930 u32 prioreg = highprio ? D40_DREG_PSEG1 : D40_DREG_PCEG1; 1931 u32 rtreg = realtime ? D40_DREG_RSEG1 : D40_DREG_RCEG1; 1932 u32 event = D40_TYPE_TO_EVENT(dev_type); 1933 u32 group = D40_TYPE_TO_GROUP(dev_type); 1934 u32 bit = 1 << event; 1935 1936 /* Destination event lines are stored in the upper halfword */ 1937 if (!src) 1938 bit <<= 16; 1939 1940 writel(bit, d40c->base->virtbase + prioreg + group * 4); 1941 writel(bit, d40c->base->virtbase + rtreg + group * 4); 1942 } 1943 1944 static void d40_set_prio_realtime(struct d40_chan *d40c) 1945 { 1946 if (d40c->base->rev < 3) 1947 return; 1948 1949 if ((d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) || 1950 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) 1951 __d40_set_prio_rt(d40c, d40c->dma_cfg.src_dev_type, true); 1952 1953 if ((d40c->dma_cfg.dir == STEDMA40_MEM_TO_PERIPH) || 1954 (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_PERIPH)) 1955 __d40_set_prio_rt(d40c, d40c->dma_cfg.dst_dev_type, false); 1956 } 1957 1958 /* DMA ENGINE functions */ 1959 static int d40_alloc_chan_resources(struct dma_chan *chan) 1960 { 1961 int err; 1962 unsigned long flags; 1963 struct d40_chan *d40c = 1964 container_of(chan, struct d40_chan, chan); 1965 bool is_free_phy; 1966 spin_lock_irqsave(&d40c->lock, flags); 1967 1968 d40c->completed = chan->cookie = 1; 1969 1970 /* If no dma configuration is set use default configuration (memcpy) */ 1971 if (!d40c->configured) { 1972 err = d40_config_memcpy(d40c); 1973 if (err) { 1974 chan_err(d40c, "Failed to configure memcpy channel\n"); 1975 goto fail; 1976 } 1977 } 1978 is_free_phy = (d40c->phy_chan == NULL); 1979 1980 err = d40_allocate_channel(d40c); 1981 if (err) { 1982 chan_err(d40c, "Failed to allocate channel\n"); 1983 goto fail; 1984 } 1985 1986 /* Fill in basic CFG register values */ 1987 d40_phy_cfg(&d40c->dma_cfg, &d40c->src_def_cfg, 1988 &d40c->dst_def_cfg, chan_is_logical(d40c)); 1989 1990 d40_set_prio_realtime(d40c); 1991 1992 if (chan_is_logical(d40c)) { 1993 d40_log_cfg(&d40c->dma_cfg, 1994 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); 1995 1996 if (d40c->dma_cfg.dir == STEDMA40_PERIPH_TO_MEM) 1997 d40c->lcpa = d40c->base->lcpa_base + 1998 d40c->dma_cfg.src_dev_type * D40_LCPA_CHAN_SIZE; 1999 else 2000 d40c->lcpa = d40c->base->lcpa_base + 2001 d40c->dma_cfg.dst_dev_type * 2002 D40_LCPA_CHAN_SIZE + D40_LCPA_CHAN_DST_DELTA; 2003 } 2004 2005 /* 2006 * Only write channel configuration to the DMA if the physical 2007 * resource is free. In case of multiple logical channels 2008 * on the same physical resource, only the first write is necessary. 2009 */ 2010 if (is_free_phy) 2011 d40_config_write(d40c); 2012 fail: 2013 spin_unlock_irqrestore(&d40c->lock, flags); 2014 return err; 2015 } 2016 2017 static void d40_free_chan_resources(struct dma_chan *chan) 2018 { 2019 struct d40_chan *d40c = 2020 container_of(chan, struct d40_chan, chan); 2021 int err; 2022 unsigned long flags; 2023 2024 if (d40c->phy_chan == NULL) { 2025 chan_err(d40c, "Cannot free unallocated channel\n"); 2026 return; 2027 } 2028 2029 2030 spin_lock_irqsave(&d40c->lock, flags); 2031 2032 err = d40_free_dma(d40c); 2033 2034 if (err) 2035 chan_err(d40c, "Failed to free channel\n"); 2036 spin_unlock_irqrestore(&d40c->lock, flags); 2037 } 2038 2039 static struct dma_async_tx_descriptor *d40_prep_memcpy(struct dma_chan *chan, 2040 dma_addr_t dst, 2041 dma_addr_t src, 2042 size_t size, 2043 unsigned long dma_flags) 2044 { 2045 struct scatterlist dst_sg; 2046 struct scatterlist src_sg; 2047 2048 sg_init_table(&dst_sg, 1); 2049 sg_init_table(&src_sg, 1); 2050 2051 sg_dma_address(&dst_sg) = dst; 2052 sg_dma_address(&src_sg) = src; 2053 2054 sg_dma_len(&dst_sg) = size; 2055 sg_dma_len(&src_sg) = size; 2056 2057 return d40_prep_sg(chan, &src_sg, &dst_sg, 1, DMA_NONE, dma_flags); 2058 } 2059 2060 static struct dma_async_tx_descriptor * 2061 d40_prep_memcpy_sg(struct dma_chan *chan, 2062 struct scatterlist *dst_sg, unsigned int dst_nents, 2063 struct scatterlist *src_sg, unsigned int src_nents, 2064 unsigned long dma_flags) 2065 { 2066 if (dst_nents != src_nents) 2067 return NULL; 2068 2069 return d40_prep_sg(chan, src_sg, dst_sg, src_nents, DMA_NONE, dma_flags); 2070 } 2071 2072 static struct dma_async_tx_descriptor *d40_prep_slave_sg(struct dma_chan *chan, 2073 struct scatterlist *sgl, 2074 unsigned int sg_len, 2075 enum dma_data_direction direction, 2076 unsigned long dma_flags) 2077 { 2078 if (direction != DMA_FROM_DEVICE && direction != DMA_TO_DEVICE) 2079 return NULL; 2080 2081 return d40_prep_sg(chan, sgl, sgl, sg_len, direction, dma_flags); 2082 } 2083 2084 static struct dma_async_tx_descriptor * 2085 dma40_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr, 2086 size_t buf_len, size_t period_len, 2087 enum dma_data_direction direction) 2088 { 2089 unsigned int periods = buf_len / period_len; 2090 struct dma_async_tx_descriptor *txd; 2091 struct scatterlist *sg; 2092 int i; 2093 2094 sg = kcalloc(periods + 1, sizeof(struct scatterlist), GFP_KERNEL); 2095 for (i = 0; i < periods; i++) { 2096 sg_dma_address(&sg[i]) = dma_addr; 2097 sg_dma_len(&sg[i]) = period_len; 2098 dma_addr += period_len; 2099 } 2100 2101 sg[periods].offset = 0; 2102 sg[periods].length = 0; 2103 sg[periods].page_link = 2104 ((unsigned long)sg | 0x01) & ~0x02; 2105 2106 txd = d40_prep_sg(chan, sg, sg, periods, direction, 2107 DMA_PREP_INTERRUPT); 2108 2109 kfree(sg); 2110 2111 return txd; 2112 } 2113 2114 static enum dma_status d40_tx_status(struct dma_chan *chan, 2115 dma_cookie_t cookie, 2116 struct dma_tx_state *txstate) 2117 { 2118 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2119 dma_cookie_t last_used; 2120 dma_cookie_t last_complete; 2121 int ret; 2122 2123 if (d40c->phy_chan == NULL) { 2124 chan_err(d40c, "Cannot read status of unallocated channel\n"); 2125 return -EINVAL; 2126 } 2127 2128 last_complete = d40c->completed; 2129 last_used = chan->cookie; 2130 2131 if (d40_is_paused(d40c)) 2132 ret = DMA_PAUSED; 2133 else 2134 ret = dma_async_is_complete(cookie, last_complete, last_used); 2135 2136 dma_set_tx_state(txstate, last_complete, last_used, 2137 stedma40_residue(chan)); 2138 2139 return ret; 2140 } 2141 2142 static void d40_issue_pending(struct dma_chan *chan) 2143 { 2144 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2145 unsigned long flags; 2146 2147 if (d40c->phy_chan == NULL) { 2148 chan_err(d40c, "Channel is not allocated!\n"); 2149 return; 2150 } 2151 2152 spin_lock_irqsave(&d40c->lock, flags); 2153 2154 /* Busy means that pending jobs are already being processed */ 2155 if (!d40c->busy) 2156 (void) d40_queue_start(d40c); 2157 2158 spin_unlock_irqrestore(&d40c->lock, flags); 2159 } 2160 2161 /* Runtime reconfiguration extension */ 2162 static void d40_set_runtime_config(struct dma_chan *chan, 2163 struct dma_slave_config *config) 2164 { 2165 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2166 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg; 2167 enum dma_slave_buswidth config_addr_width; 2168 dma_addr_t config_addr; 2169 u32 config_maxburst; 2170 enum stedma40_periph_data_width addr_width; 2171 int psize; 2172 2173 if (config->direction == DMA_FROM_DEVICE) { 2174 dma_addr_t dev_addr_rx = 2175 d40c->base->plat_data->dev_rx[cfg->src_dev_type]; 2176 2177 config_addr = config->src_addr; 2178 if (dev_addr_rx) 2179 dev_dbg(d40c->base->dev, 2180 "channel has a pre-wired RX address %08x " 2181 "overriding with %08x\n", 2182 dev_addr_rx, config_addr); 2183 if (cfg->dir != STEDMA40_PERIPH_TO_MEM) 2184 dev_dbg(d40c->base->dev, 2185 "channel was not configured for peripheral " 2186 "to memory transfer (%d) overriding\n", 2187 cfg->dir); 2188 cfg->dir = STEDMA40_PERIPH_TO_MEM; 2189 2190 config_addr_width = config->src_addr_width; 2191 config_maxburst = config->src_maxburst; 2192 2193 } else if (config->direction == DMA_TO_DEVICE) { 2194 dma_addr_t dev_addr_tx = 2195 d40c->base->plat_data->dev_tx[cfg->dst_dev_type]; 2196 2197 config_addr = config->dst_addr; 2198 if (dev_addr_tx) 2199 dev_dbg(d40c->base->dev, 2200 "channel has a pre-wired TX address %08x " 2201 "overriding with %08x\n", 2202 dev_addr_tx, config_addr); 2203 if (cfg->dir != STEDMA40_MEM_TO_PERIPH) 2204 dev_dbg(d40c->base->dev, 2205 "channel was not configured for memory " 2206 "to peripheral transfer (%d) overriding\n", 2207 cfg->dir); 2208 cfg->dir = STEDMA40_MEM_TO_PERIPH; 2209 2210 config_addr_width = config->dst_addr_width; 2211 config_maxburst = config->dst_maxburst; 2212 2213 } else { 2214 dev_err(d40c->base->dev, 2215 "unrecognized channel direction %d\n", 2216 config->direction); 2217 return; 2218 } 2219 2220 switch (config_addr_width) { 2221 case DMA_SLAVE_BUSWIDTH_1_BYTE: 2222 addr_width = STEDMA40_BYTE_WIDTH; 2223 break; 2224 case DMA_SLAVE_BUSWIDTH_2_BYTES: 2225 addr_width = STEDMA40_HALFWORD_WIDTH; 2226 break; 2227 case DMA_SLAVE_BUSWIDTH_4_BYTES: 2228 addr_width = STEDMA40_WORD_WIDTH; 2229 break; 2230 case DMA_SLAVE_BUSWIDTH_8_BYTES: 2231 addr_width = STEDMA40_DOUBLEWORD_WIDTH; 2232 break; 2233 default: 2234 dev_err(d40c->base->dev, 2235 "illegal peripheral address width " 2236 "requested (%d)\n", 2237 config->src_addr_width); 2238 return; 2239 } 2240 2241 if (chan_is_logical(d40c)) { 2242 if (config_maxburst >= 16) 2243 psize = STEDMA40_PSIZE_LOG_16; 2244 else if (config_maxburst >= 8) 2245 psize = STEDMA40_PSIZE_LOG_8; 2246 else if (config_maxburst >= 4) 2247 psize = STEDMA40_PSIZE_LOG_4; 2248 else 2249 psize = STEDMA40_PSIZE_LOG_1; 2250 } else { 2251 if (config_maxburst >= 16) 2252 psize = STEDMA40_PSIZE_PHY_16; 2253 else if (config_maxburst >= 8) 2254 psize = STEDMA40_PSIZE_PHY_8; 2255 else if (config_maxburst >= 4) 2256 psize = STEDMA40_PSIZE_PHY_4; 2257 else if (config_maxburst >= 2) 2258 psize = STEDMA40_PSIZE_PHY_2; 2259 else 2260 psize = STEDMA40_PSIZE_PHY_1; 2261 } 2262 2263 /* Set up all the endpoint configs */ 2264 cfg->src_info.data_width = addr_width; 2265 cfg->src_info.psize = psize; 2266 cfg->src_info.big_endian = false; 2267 cfg->src_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL; 2268 cfg->dst_info.data_width = addr_width; 2269 cfg->dst_info.psize = psize; 2270 cfg->dst_info.big_endian = false; 2271 cfg->dst_info.flow_ctrl = STEDMA40_NO_FLOW_CTRL; 2272 2273 /* Fill in register values */ 2274 if (chan_is_logical(d40c)) 2275 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); 2276 else 2277 d40_phy_cfg(cfg, &d40c->src_def_cfg, 2278 &d40c->dst_def_cfg, false); 2279 2280 /* These settings will take precedence later */ 2281 d40c->runtime_addr = config_addr; 2282 d40c->runtime_direction = config->direction; 2283 dev_dbg(d40c->base->dev, 2284 "configured channel %s for %s, data width %d, " 2285 "maxburst %d bytes, LE, no flow control\n", 2286 dma_chan_name(chan), 2287 (config->direction == DMA_FROM_DEVICE) ? "RX" : "TX", 2288 config_addr_width, 2289 config_maxburst); 2290 } 2291 2292 static int d40_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd, 2293 unsigned long arg) 2294 { 2295 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); 2296 2297 if (d40c->phy_chan == NULL) { 2298 chan_err(d40c, "Channel is not allocated!\n"); 2299 return -EINVAL; 2300 } 2301 2302 switch (cmd) { 2303 case DMA_TERMINATE_ALL: 2304 return d40_terminate_all(d40c); 2305 case DMA_PAUSE: 2306 return d40_pause(d40c); 2307 case DMA_RESUME: 2308 return d40_resume(d40c); 2309 case DMA_SLAVE_CONFIG: 2310 d40_set_runtime_config(chan, 2311 (struct dma_slave_config *) arg); 2312 return 0; 2313 default: 2314 break; 2315 } 2316 2317 /* Other commands are unimplemented */ 2318 return -ENXIO; 2319 } 2320 2321 /* Initialization functions */ 2322 2323 static void __init d40_chan_init(struct d40_base *base, struct dma_device *dma, 2324 struct d40_chan *chans, int offset, 2325 int num_chans) 2326 { 2327 int i = 0; 2328 struct d40_chan *d40c; 2329 2330 INIT_LIST_HEAD(&dma->channels); 2331 2332 for (i = offset; i < offset + num_chans; i++) { 2333 d40c = &chans[i]; 2334 d40c->base = base; 2335 d40c->chan.device = dma; 2336 2337 spin_lock_init(&d40c->lock); 2338 2339 d40c->log_num = D40_PHY_CHAN; 2340 2341 INIT_LIST_HEAD(&d40c->active); 2342 INIT_LIST_HEAD(&d40c->queue); 2343 INIT_LIST_HEAD(&d40c->client); 2344 2345 tasklet_init(&d40c->tasklet, dma_tasklet, 2346 (unsigned long) d40c); 2347 2348 list_add_tail(&d40c->chan.device_node, 2349 &dma->channels); 2350 } 2351 } 2352 2353 static void d40_ops_init(struct d40_base *base, struct dma_device *dev) 2354 { 2355 if (dma_has_cap(DMA_SLAVE, dev->cap_mask)) 2356 dev->device_prep_slave_sg = d40_prep_slave_sg; 2357 2358 if (dma_has_cap(DMA_MEMCPY, dev->cap_mask)) { 2359 dev->device_prep_dma_memcpy = d40_prep_memcpy; 2360 2361 /* 2362 * This controller can only access address at even 2363 * 32bit boundaries, i.e. 2^2 2364 */ 2365 dev->copy_align = 2; 2366 } 2367 2368 if (dma_has_cap(DMA_SG, dev->cap_mask)) 2369 dev->device_prep_dma_sg = d40_prep_memcpy_sg; 2370 2371 if (dma_has_cap(DMA_CYCLIC, dev->cap_mask)) 2372 dev->device_prep_dma_cyclic = dma40_prep_dma_cyclic; 2373 2374 dev->device_alloc_chan_resources = d40_alloc_chan_resources; 2375 dev->device_free_chan_resources = d40_free_chan_resources; 2376 dev->device_issue_pending = d40_issue_pending; 2377 dev->device_tx_status = d40_tx_status; 2378 dev->device_control = d40_control; 2379 dev->dev = base->dev; 2380 } 2381 2382 static int __init d40_dmaengine_init(struct d40_base *base, 2383 int num_reserved_chans) 2384 { 2385 int err ; 2386 2387 d40_chan_init(base, &base->dma_slave, base->log_chans, 2388 0, base->num_log_chans); 2389 2390 dma_cap_zero(base->dma_slave.cap_mask); 2391 dma_cap_set(DMA_SLAVE, base->dma_slave.cap_mask); 2392 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); 2393 2394 d40_ops_init(base, &base->dma_slave); 2395 2396 err = dma_async_device_register(&base->dma_slave); 2397 2398 if (err) { 2399 d40_err(base->dev, "Failed to register slave channels\n"); 2400 goto failure1; 2401 } 2402 2403 d40_chan_init(base, &base->dma_memcpy, base->log_chans, 2404 base->num_log_chans, base->plat_data->memcpy_len); 2405 2406 dma_cap_zero(base->dma_memcpy.cap_mask); 2407 dma_cap_set(DMA_MEMCPY, base->dma_memcpy.cap_mask); 2408 dma_cap_set(DMA_SG, base->dma_memcpy.cap_mask); 2409 2410 d40_ops_init(base, &base->dma_memcpy); 2411 2412 err = dma_async_device_register(&base->dma_memcpy); 2413 2414 if (err) { 2415 d40_err(base->dev, 2416 "Failed to regsiter memcpy only channels\n"); 2417 goto failure2; 2418 } 2419 2420 d40_chan_init(base, &base->dma_both, base->phy_chans, 2421 0, num_reserved_chans); 2422 2423 dma_cap_zero(base->dma_both.cap_mask); 2424 dma_cap_set(DMA_SLAVE, base->dma_both.cap_mask); 2425 dma_cap_set(DMA_MEMCPY, base->dma_both.cap_mask); 2426 dma_cap_set(DMA_SG, base->dma_both.cap_mask); 2427 dma_cap_set(DMA_CYCLIC, base->dma_slave.cap_mask); 2428 2429 d40_ops_init(base, &base->dma_both); 2430 err = dma_async_device_register(&base->dma_both); 2431 2432 if (err) { 2433 d40_err(base->dev, 2434 "Failed to register logical and physical capable channels\n"); 2435 goto failure3; 2436 } 2437 return 0; 2438 failure3: 2439 dma_async_device_unregister(&base->dma_memcpy); 2440 failure2: 2441 dma_async_device_unregister(&base->dma_slave); 2442 failure1: 2443 return err; 2444 } 2445 2446 /* Initialization functions. */ 2447 2448 static int __init d40_phy_res_init(struct d40_base *base) 2449 { 2450 int i; 2451 int num_phy_chans_avail = 0; 2452 u32 val[2]; 2453 int odd_even_bit = -2; 2454 2455 val[0] = readl(base->virtbase + D40_DREG_PRSME); 2456 val[1] = readl(base->virtbase + D40_DREG_PRSMO); 2457 2458 for (i = 0; i < base->num_phy_chans; i++) { 2459 base->phy_res[i].num = i; 2460 odd_even_bit += 2 * ((i % 2) == 0); 2461 if (((val[i % 2] >> odd_even_bit) & 3) == 1) { 2462 /* Mark security only channels as occupied */ 2463 base->phy_res[i].allocated_src = D40_ALLOC_PHY; 2464 base->phy_res[i].allocated_dst = D40_ALLOC_PHY; 2465 } else { 2466 base->phy_res[i].allocated_src = D40_ALLOC_FREE; 2467 base->phy_res[i].allocated_dst = D40_ALLOC_FREE; 2468 num_phy_chans_avail++; 2469 } 2470 spin_lock_init(&base->phy_res[i].lock); 2471 } 2472 2473 /* Mark disabled channels as occupied */ 2474 for (i = 0; base->plat_data->disabled_channels[i] != -1; i++) { 2475 int chan = base->plat_data->disabled_channels[i]; 2476 2477 base->phy_res[chan].allocated_src = D40_ALLOC_PHY; 2478 base->phy_res[chan].allocated_dst = D40_ALLOC_PHY; 2479 num_phy_chans_avail--; 2480 } 2481 2482 dev_info(base->dev, "%d of %d physical DMA channels available\n", 2483 num_phy_chans_avail, base->num_phy_chans); 2484 2485 /* Verify settings extended vs standard */ 2486 val[0] = readl(base->virtbase + D40_DREG_PRTYP); 2487 2488 for (i = 0; i < base->num_phy_chans; i++) { 2489 2490 if (base->phy_res[i].allocated_src == D40_ALLOC_FREE && 2491 (val[0] & 0x3) != 1) 2492 dev_info(base->dev, 2493 "[%s] INFO: channel %d is misconfigured (%d)\n", 2494 __func__, i, val[0] & 0x3); 2495 2496 val[0] = val[0] >> 2; 2497 } 2498 2499 return num_phy_chans_avail; 2500 } 2501 2502 static struct d40_base * __init d40_hw_detect_init(struct platform_device *pdev) 2503 { 2504 static const struct d40_reg_val dma_id_regs[] = { 2505 /* Peripheral Id */ 2506 { .reg = D40_DREG_PERIPHID0, .val = 0x0040}, 2507 { .reg = D40_DREG_PERIPHID1, .val = 0x0000}, 2508 /* 2509 * D40_DREG_PERIPHID2 Depends on HW revision: 2510 * DB8500ed has 0x0008, 2511 * ? has 0x0018, 2512 * DB8500v1 has 0x0028 2513 * DB8500v2 has 0x0038 2514 */ 2515 { .reg = D40_DREG_PERIPHID3, .val = 0x0000}, 2516 2517 /* PCell Id */ 2518 { .reg = D40_DREG_CELLID0, .val = 0x000d}, 2519 { .reg = D40_DREG_CELLID1, .val = 0x00f0}, 2520 { .reg = D40_DREG_CELLID2, .val = 0x0005}, 2521 { .reg = D40_DREG_CELLID3, .val = 0x00b1} 2522 }; 2523 struct stedma40_platform_data *plat_data; 2524 struct clk *clk = NULL; 2525 void __iomem *virtbase = NULL; 2526 struct resource *res = NULL; 2527 struct d40_base *base = NULL; 2528 int num_log_chans = 0; 2529 int num_phy_chans; 2530 int i; 2531 u32 val; 2532 u32 rev; 2533 2534 clk = clk_get(&pdev->dev, NULL); 2535 2536 if (IS_ERR(clk)) { 2537 d40_err(&pdev->dev, "No matching clock found\n"); 2538 goto failure; 2539 } 2540 2541 clk_enable(clk); 2542 2543 /* Get IO for DMAC base address */ 2544 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base"); 2545 if (!res) 2546 goto failure; 2547 2548 if (request_mem_region(res->start, resource_size(res), 2549 D40_NAME " I/O base") == NULL) 2550 goto failure; 2551 2552 virtbase = ioremap(res->start, resource_size(res)); 2553 if (!virtbase) 2554 goto failure; 2555 2556 /* HW version check */ 2557 for (i = 0; i < ARRAY_SIZE(dma_id_regs); i++) { 2558 if (dma_id_regs[i].val != 2559 readl(virtbase + dma_id_regs[i].reg)) { 2560 d40_err(&pdev->dev, 2561 "Unknown hardware! Expected 0x%x at 0x%x but got 0x%x\n", 2562 dma_id_regs[i].val, 2563 dma_id_regs[i].reg, 2564 readl(virtbase + dma_id_regs[i].reg)); 2565 goto failure; 2566 } 2567 } 2568 2569 /* Get silicon revision and designer */ 2570 val = readl(virtbase + D40_DREG_PERIPHID2); 2571 2572 if ((val & D40_DREG_PERIPHID2_DESIGNER_MASK) != 2573 D40_HW_DESIGNER) { 2574 d40_err(&pdev->dev, "Unknown designer! Got %x wanted %x\n", 2575 val & D40_DREG_PERIPHID2_DESIGNER_MASK, 2576 D40_HW_DESIGNER); 2577 goto failure; 2578 } 2579 2580 rev = (val & D40_DREG_PERIPHID2_REV_MASK) >> 2581 D40_DREG_PERIPHID2_REV_POS; 2582 2583 /* The number of physical channels on this HW */ 2584 num_phy_chans = 4 * (readl(virtbase + D40_DREG_ICFG) & 0x7) + 4; 2585 2586 dev_info(&pdev->dev, "hardware revision: %d @ 0x%x\n", 2587 rev, res->start); 2588 2589 plat_data = pdev->dev.platform_data; 2590 2591 /* Count the number of logical channels in use */ 2592 for (i = 0; i < plat_data->dev_len; i++) 2593 if (plat_data->dev_rx[i] != 0) 2594 num_log_chans++; 2595 2596 for (i = 0; i < plat_data->dev_len; i++) 2597 if (plat_data->dev_tx[i] != 0) 2598 num_log_chans++; 2599 2600 base = kzalloc(ALIGN(sizeof(struct d40_base), 4) + 2601 (num_phy_chans + num_log_chans + plat_data->memcpy_len) * 2602 sizeof(struct d40_chan), GFP_KERNEL); 2603 2604 if (base == NULL) { 2605 d40_err(&pdev->dev, "Out of memory\n"); 2606 goto failure; 2607 } 2608 2609 base->rev = rev; 2610 base->clk = clk; 2611 base->num_phy_chans = num_phy_chans; 2612 base->num_log_chans = num_log_chans; 2613 base->phy_start = res->start; 2614 base->phy_size = resource_size(res); 2615 base->virtbase = virtbase; 2616 base->plat_data = plat_data; 2617 base->dev = &pdev->dev; 2618 base->phy_chans = ((void *)base) + ALIGN(sizeof(struct d40_base), 4); 2619 base->log_chans = &base->phy_chans[num_phy_chans]; 2620 2621 base->phy_res = kzalloc(num_phy_chans * sizeof(struct d40_phy_res), 2622 GFP_KERNEL); 2623 if (!base->phy_res) 2624 goto failure; 2625 2626 base->lookup_phy_chans = kzalloc(num_phy_chans * 2627 sizeof(struct d40_chan *), 2628 GFP_KERNEL); 2629 if (!base->lookup_phy_chans) 2630 goto failure; 2631 2632 if (num_log_chans + plat_data->memcpy_len) { 2633 /* 2634 * The max number of logical channels are event lines for all 2635 * src devices and dst devices 2636 */ 2637 base->lookup_log_chans = kzalloc(plat_data->dev_len * 2 * 2638 sizeof(struct d40_chan *), 2639 GFP_KERNEL); 2640 if (!base->lookup_log_chans) 2641 goto failure; 2642 } 2643 2644 base->lcla_pool.alloc_map = kzalloc(num_phy_chans * 2645 sizeof(struct d40_desc *) * 2646 D40_LCLA_LINK_PER_EVENT_GRP, 2647 GFP_KERNEL); 2648 if (!base->lcla_pool.alloc_map) 2649 goto failure; 2650 2651 base->desc_slab = kmem_cache_create(D40_NAME, sizeof(struct d40_desc), 2652 0, SLAB_HWCACHE_ALIGN, 2653 NULL); 2654 if (base->desc_slab == NULL) 2655 goto failure; 2656 2657 return base; 2658 2659 failure: 2660 if (!IS_ERR(clk)) { 2661 clk_disable(clk); 2662 clk_put(clk); 2663 } 2664 if (virtbase) 2665 iounmap(virtbase); 2666 if (res) 2667 release_mem_region(res->start, 2668 resource_size(res)); 2669 if (virtbase) 2670 iounmap(virtbase); 2671 2672 if (base) { 2673 kfree(base->lcla_pool.alloc_map); 2674 kfree(base->lookup_log_chans); 2675 kfree(base->lookup_phy_chans); 2676 kfree(base->phy_res); 2677 kfree(base); 2678 } 2679 2680 return NULL; 2681 } 2682 2683 static void __init d40_hw_init(struct d40_base *base) 2684 { 2685 2686 static const struct d40_reg_val dma_init_reg[] = { 2687 /* Clock every part of the DMA block from start */ 2688 { .reg = D40_DREG_GCC, .val = 0x0000ff01}, 2689 2690 /* Interrupts on all logical channels */ 2691 { .reg = D40_DREG_LCMIS0, .val = 0xFFFFFFFF}, 2692 { .reg = D40_DREG_LCMIS1, .val = 0xFFFFFFFF}, 2693 { .reg = D40_DREG_LCMIS2, .val = 0xFFFFFFFF}, 2694 { .reg = D40_DREG_LCMIS3, .val = 0xFFFFFFFF}, 2695 { .reg = D40_DREG_LCICR0, .val = 0xFFFFFFFF}, 2696 { .reg = D40_DREG_LCICR1, .val = 0xFFFFFFFF}, 2697 { .reg = D40_DREG_LCICR2, .val = 0xFFFFFFFF}, 2698 { .reg = D40_DREG_LCICR3, .val = 0xFFFFFFFF}, 2699 { .reg = D40_DREG_LCTIS0, .val = 0xFFFFFFFF}, 2700 { .reg = D40_DREG_LCTIS1, .val = 0xFFFFFFFF}, 2701 { .reg = D40_DREG_LCTIS2, .val = 0xFFFFFFFF}, 2702 { .reg = D40_DREG_LCTIS3, .val = 0xFFFFFFFF} 2703 }; 2704 int i; 2705 u32 prmseo[2] = {0, 0}; 2706 u32 activeo[2] = {0xFFFFFFFF, 0xFFFFFFFF}; 2707 u32 pcmis = 0; 2708 u32 pcicr = 0; 2709 2710 for (i = 0; i < ARRAY_SIZE(dma_init_reg); i++) 2711 writel(dma_init_reg[i].val, 2712 base->virtbase + dma_init_reg[i].reg); 2713 2714 /* Configure all our dma channels to default settings */ 2715 for (i = 0; i < base->num_phy_chans; i++) { 2716 2717 activeo[i % 2] = activeo[i % 2] << 2; 2718 2719 if (base->phy_res[base->num_phy_chans - i - 1].allocated_src 2720 == D40_ALLOC_PHY) { 2721 activeo[i % 2] |= 3; 2722 continue; 2723 } 2724 2725 /* Enable interrupt # */ 2726 pcmis = (pcmis << 1) | 1; 2727 2728 /* Clear interrupt # */ 2729 pcicr = (pcicr << 1) | 1; 2730 2731 /* Set channel to physical mode */ 2732 prmseo[i % 2] = prmseo[i % 2] << 2; 2733 prmseo[i % 2] |= 1; 2734 2735 } 2736 2737 writel(prmseo[1], base->virtbase + D40_DREG_PRMSE); 2738 writel(prmseo[0], base->virtbase + D40_DREG_PRMSO); 2739 writel(activeo[1], base->virtbase + D40_DREG_ACTIVE); 2740 writel(activeo[0], base->virtbase + D40_DREG_ACTIVO); 2741 2742 /* Write which interrupt to enable */ 2743 writel(pcmis, base->virtbase + D40_DREG_PCMIS); 2744 2745 /* Write which interrupt to clear */ 2746 writel(pcicr, base->virtbase + D40_DREG_PCICR); 2747 2748 } 2749 2750 static int __init d40_lcla_allocate(struct d40_base *base) 2751 { 2752 struct d40_lcla_pool *pool = &base->lcla_pool; 2753 unsigned long *page_list; 2754 int i, j; 2755 int ret = 0; 2756 2757 /* 2758 * This is somewhat ugly. We need 8192 bytes that are 18 bit aligned, 2759 * To full fill this hardware requirement without wasting 256 kb 2760 * we allocate pages until we get an aligned one. 2761 */ 2762 page_list = kmalloc(sizeof(unsigned long) * MAX_LCLA_ALLOC_ATTEMPTS, 2763 GFP_KERNEL); 2764 2765 if (!page_list) { 2766 ret = -ENOMEM; 2767 goto failure; 2768 } 2769 2770 /* Calculating how many pages that are required */ 2771 base->lcla_pool.pages = SZ_1K * base->num_phy_chans / PAGE_SIZE; 2772 2773 for (i = 0; i < MAX_LCLA_ALLOC_ATTEMPTS; i++) { 2774 page_list[i] = __get_free_pages(GFP_KERNEL, 2775 base->lcla_pool.pages); 2776 if (!page_list[i]) { 2777 2778 d40_err(base->dev, "Failed to allocate %d pages.\n", 2779 base->lcla_pool.pages); 2780 2781 for (j = 0; j < i; j++) 2782 free_pages(page_list[j], base->lcla_pool.pages); 2783 goto failure; 2784 } 2785 2786 if ((virt_to_phys((void *)page_list[i]) & 2787 (LCLA_ALIGNMENT - 1)) == 0) 2788 break; 2789 } 2790 2791 for (j = 0; j < i; j++) 2792 free_pages(page_list[j], base->lcla_pool.pages); 2793 2794 if (i < MAX_LCLA_ALLOC_ATTEMPTS) { 2795 base->lcla_pool.base = (void *)page_list[i]; 2796 } else { 2797 /* 2798 * After many attempts and no succees with finding the correct 2799 * alignment, try with allocating a big buffer. 2800 */ 2801 dev_warn(base->dev, 2802 "[%s] Failed to get %d pages @ 18 bit align.\n", 2803 __func__, base->lcla_pool.pages); 2804 base->lcla_pool.base_unaligned = kmalloc(SZ_1K * 2805 base->num_phy_chans + 2806 LCLA_ALIGNMENT, 2807 GFP_KERNEL); 2808 if (!base->lcla_pool.base_unaligned) { 2809 ret = -ENOMEM; 2810 goto failure; 2811 } 2812 2813 base->lcla_pool.base = PTR_ALIGN(base->lcla_pool.base_unaligned, 2814 LCLA_ALIGNMENT); 2815 } 2816 2817 pool->dma_addr = dma_map_single(base->dev, pool->base, 2818 SZ_1K * base->num_phy_chans, 2819 DMA_TO_DEVICE); 2820 if (dma_mapping_error(base->dev, pool->dma_addr)) { 2821 pool->dma_addr = 0; 2822 ret = -ENOMEM; 2823 goto failure; 2824 } 2825 2826 writel(virt_to_phys(base->lcla_pool.base), 2827 base->virtbase + D40_DREG_LCLA); 2828 failure: 2829 kfree(page_list); 2830 return ret; 2831 } 2832 2833 static int __init d40_probe(struct platform_device *pdev) 2834 { 2835 int err; 2836 int ret = -ENOENT; 2837 struct d40_base *base; 2838 struct resource *res = NULL; 2839 int num_reserved_chans; 2840 u32 val; 2841 2842 base = d40_hw_detect_init(pdev); 2843 2844 if (!base) 2845 goto failure; 2846 2847 num_reserved_chans = d40_phy_res_init(base); 2848 2849 platform_set_drvdata(pdev, base); 2850 2851 spin_lock_init(&base->interrupt_lock); 2852 spin_lock_init(&base->execmd_lock); 2853 2854 /* Get IO for logical channel parameter address */ 2855 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "lcpa"); 2856 if (!res) { 2857 ret = -ENOENT; 2858 d40_err(&pdev->dev, "No \"lcpa\" memory resource\n"); 2859 goto failure; 2860 } 2861 base->lcpa_size = resource_size(res); 2862 base->phy_lcpa = res->start; 2863 2864 if (request_mem_region(res->start, resource_size(res), 2865 D40_NAME " I/O lcpa") == NULL) { 2866 ret = -EBUSY; 2867 d40_err(&pdev->dev, 2868 "Failed to request LCPA region 0x%x-0x%x\n", 2869 res->start, res->end); 2870 goto failure; 2871 } 2872 2873 /* We make use of ESRAM memory for this. */ 2874 val = readl(base->virtbase + D40_DREG_LCPA); 2875 if (res->start != val && val != 0) { 2876 dev_warn(&pdev->dev, 2877 "[%s] Mismatch LCPA dma 0x%x, def 0x%x\n", 2878 __func__, val, res->start); 2879 } else 2880 writel(res->start, base->virtbase + D40_DREG_LCPA); 2881 2882 base->lcpa_base = ioremap(res->start, resource_size(res)); 2883 if (!base->lcpa_base) { 2884 ret = -ENOMEM; 2885 d40_err(&pdev->dev, "Failed to ioremap LCPA region\n"); 2886 goto failure; 2887 } 2888 2889 ret = d40_lcla_allocate(base); 2890 if (ret) { 2891 d40_err(&pdev->dev, "Failed to allocate LCLA area\n"); 2892 goto failure; 2893 } 2894 2895 spin_lock_init(&base->lcla_pool.lock); 2896 2897 base->irq = platform_get_irq(pdev, 0); 2898 2899 ret = request_irq(base->irq, d40_handle_interrupt, 0, D40_NAME, base); 2900 if (ret) { 2901 d40_err(&pdev->dev, "No IRQ defined\n"); 2902 goto failure; 2903 } 2904 2905 err = d40_dmaengine_init(base, num_reserved_chans); 2906 if (err) 2907 goto failure; 2908 2909 d40_hw_init(base); 2910 2911 dev_info(base->dev, "initialized\n"); 2912 return 0; 2913 2914 failure: 2915 if (base) { 2916 if (base->desc_slab) 2917 kmem_cache_destroy(base->desc_slab); 2918 if (base->virtbase) 2919 iounmap(base->virtbase); 2920 2921 if (base->lcla_pool.dma_addr) 2922 dma_unmap_single(base->dev, base->lcla_pool.dma_addr, 2923 SZ_1K * base->num_phy_chans, 2924 DMA_TO_DEVICE); 2925 2926 if (!base->lcla_pool.base_unaligned && base->lcla_pool.base) 2927 free_pages((unsigned long)base->lcla_pool.base, 2928 base->lcla_pool.pages); 2929 2930 kfree(base->lcla_pool.base_unaligned); 2931 2932 if (base->phy_lcpa) 2933 release_mem_region(base->phy_lcpa, 2934 base->lcpa_size); 2935 if (base->phy_start) 2936 release_mem_region(base->phy_start, 2937 base->phy_size); 2938 if (base->clk) { 2939 clk_disable(base->clk); 2940 clk_put(base->clk); 2941 } 2942 2943 kfree(base->lcla_pool.alloc_map); 2944 kfree(base->lookup_log_chans); 2945 kfree(base->lookup_phy_chans); 2946 kfree(base->phy_res); 2947 kfree(base); 2948 } 2949 2950 d40_err(&pdev->dev, "probe failed\n"); 2951 return ret; 2952 } 2953 2954 static struct platform_driver d40_driver = { 2955 .driver = { 2956 .owner = THIS_MODULE, 2957 .name = D40_NAME, 2958 }, 2959 }; 2960 2961 static int __init stedma40_init(void) 2962 { 2963 return platform_driver_probe(&d40_driver, d40_probe); 2964 } 2965 subsys_initcall(stedma40_init); 2966