1 /* 2 * Qualcomm Technologies HIDMA DMA engine interface 3 * 4 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 and 8 * only version 2 as published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 */ 15 16 /* 17 * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008. 18 * Copyright (C) Semihalf 2009 19 * Copyright (C) Ilya Yanok, Emcraft Systems 2010 20 * Copyright (C) Alexander Popov, Promcontroller 2014 21 * 22 * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description 23 * (defines, structures and comments) was taken from MPC5121 DMA driver 24 * written by Hongjun Chen <hong-jun.chen@freescale.com>. 25 * 26 * Approved as OSADL project by a majority of OSADL members and funded 27 * by OSADL membership fees in 2009; for details see www.osadl.org. 28 * 29 * This program is free software; you can redistribute it and/or modify it 30 * under the terms of the GNU General Public License as published by the Free 31 * Software Foundation; either version 2 of the License, or (at your option) 32 * any later version. 33 * 34 * This program is distributed in the hope that it will be useful, but WITHOUT 35 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 36 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 37 * more details. 38 * 39 * The full GNU General Public License is included in this distribution in the 40 * file called COPYING. 41 */ 42 43 /* Linux Foundation elects GPLv2 license only. */ 44 45 #include <linux/dmaengine.h> 46 #include <linux/dma-mapping.h> 47 #include <linux/list.h> 48 #include <linux/module.h> 49 #include <linux/platform_device.h> 50 #include <linux/slab.h> 51 #include <linux/spinlock.h> 52 #include <linux/property.h> 53 #include <linux/delay.h> 54 #include <linux/acpi.h> 55 #include <linux/irq.h> 56 #include <linux/atomic.h> 57 #include <linux/pm_runtime.h> 58 #include <linux/msi.h> 59 60 #include "../dmaengine.h" 61 #include "hidma.h" 62 63 /* 64 * Default idle time is 2 seconds. This parameter can 65 * be overridden by changing the following 66 * /sys/bus/platform/devices/QCOM8061:<xy>/power/autosuspend_delay_ms 67 * during kernel boot. 68 */ 69 #define HIDMA_AUTOSUSPEND_TIMEOUT 2000 70 #define HIDMA_ERR_INFO_SW 0xFF 71 #define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0 72 #define HIDMA_NR_DEFAULT_DESC 10 73 #define HIDMA_MSI_INTS 11 74 75 static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev) 76 { 77 return container_of(dmadev, struct hidma_dev, ddev); 78 } 79 80 static inline 81 struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp) 82 { 83 return container_of(_lldevp, struct hidma_dev, lldev); 84 } 85 86 static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach) 87 { 88 return container_of(dmach, struct hidma_chan, chan); 89 } 90 91 static void hidma_free(struct hidma_dev *dmadev) 92 { 93 INIT_LIST_HEAD(&dmadev->ddev.channels); 94 } 95 96 static unsigned int nr_desc_prm; 97 module_param(nr_desc_prm, uint, 0644); 98 MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)"); 99 100 enum hidma_cap { 101 HIDMA_MSI_CAP = 1, 102 HIDMA_IDENTITY_CAP, 103 }; 104 105 /* process completed descriptors */ 106 static void hidma_process_completed(struct hidma_chan *mchan) 107 { 108 struct dma_device *ddev = mchan->chan.device; 109 struct hidma_dev *mdma = to_hidma_dev(ddev); 110 struct dma_async_tx_descriptor *desc; 111 dma_cookie_t last_cookie; 112 struct hidma_desc *mdesc; 113 struct hidma_desc *next; 114 unsigned long irqflags; 115 struct list_head list; 116 117 INIT_LIST_HEAD(&list); 118 119 /* Get all completed descriptors */ 120 spin_lock_irqsave(&mchan->lock, irqflags); 121 list_splice_tail_init(&mchan->completed, &list); 122 spin_unlock_irqrestore(&mchan->lock, irqflags); 123 124 /* Execute callbacks and run dependencies */ 125 list_for_each_entry_safe(mdesc, next, &list, node) { 126 enum dma_status llstat; 127 struct dmaengine_desc_callback cb; 128 struct dmaengine_result result; 129 130 desc = &mdesc->desc; 131 last_cookie = desc->cookie; 132 133 llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); 134 135 spin_lock_irqsave(&mchan->lock, irqflags); 136 if (llstat == DMA_COMPLETE) { 137 mchan->last_success = last_cookie; 138 result.result = DMA_TRANS_NOERROR; 139 } else { 140 result.result = DMA_TRANS_ABORTED; 141 } 142 143 dma_cookie_complete(desc); 144 spin_unlock_irqrestore(&mchan->lock, irqflags); 145 146 dmaengine_desc_get_callback(desc, &cb); 147 148 dma_run_dependencies(desc); 149 150 spin_lock_irqsave(&mchan->lock, irqflags); 151 list_move(&mdesc->node, &mchan->free); 152 spin_unlock_irqrestore(&mchan->lock, irqflags); 153 154 dmaengine_desc_callback_invoke(&cb, &result); 155 } 156 } 157 158 /* 159 * Called once for each submitted descriptor. 160 * PM is locked once for each descriptor that is currently 161 * in execution. 162 */ 163 static void hidma_callback(void *data) 164 { 165 struct hidma_desc *mdesc = data; 166 struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); 167 struct dma_device *ddev = mchan->chan.device; 168 struct hidma_dev *dmadev = to_hidma_dev(ddev); 169 unsigned long irqflags; 170 bool queued = false; 171 172 spin_lock_irqsave(&mchan->lock, irqflags); 173 if (mdesc->node.next) { 174 /* Delete from the active list, add to completed list */ 175 list_move_tail(&mdesc->node, &mchan->completed); 176 queued = true; 177 178 /* calculate the next running descriptor */ 179 mchan->running = list_first_entry(&mchan->active, 180 struct hidma_desc, node); 181 } 182 spin_unlock_irqrestore(&mchan->lock, irqflags); 183 184 hidma_process_completed(mchan); 185 186 if (queued) { 187 pm_runtime_mark_last_busy(dmadev->ddev.dev); 188 pm_runtime_put_autosuspend(dmadev->ddev.dev); 189 } 190 } 191 192 static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig) 193 { 194 struct hidma_chan *mchan; 195 struct dma_device *ddev; 196 197 mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); 198 if (!mchan) 199 return -ENOMEM; 200 201 ddev = &dmadev->ddev; 202 mchan->dma_sig = dma_sig; 203 mchan->dmadev = dmadev; 204 mchan->chan.device = ddev; 205 dma_cookie_init(&mchan->chan); 206 207 INIT_LIST_HEAD(&mchan->free); 208 INIT_LIST_HEAD(&mchan->prepared); 209 INIT_LIST_HEAD(&mchan->active); 210 INIT_LIST_HEAD(&mchan->completed); 211 INIT_LIST_HEAD(&mchan->queued); 212 213 spin_lock_init(&mchan->lock); 214 list_add_tail(&mchan->chan.device_node, &ddev->channels); 215 return 0; 216 } 217 218 static void hidma_issue_task(struct tasklet_struct *t) 219 { 220 struct hidma_dev *dmadev = from_tasklet(dmadev, t, task); 221 222 pm_runtime_get_sync(dmadev->ddev.dev); 223 hidma_ll_start(dmadev->lldev); 224 } 225 226 static void hidma_issue_pending(struct dma_chan *dmach) 227 { 228 struct hidma_chan *mchan = to_hidma_chan(dmach); 229 struct hidma_dev *dmadev = mchan->dmadev; 230 unsigned long flags; 231 struct hidma_desc *qdesc, *next; 232 int status; 233 234 spin_lock_irqsave(&mchan->lock, flags); 235 list_for_each_entry_safe(qdesc, next, &mchan->queued, node) { 236 hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch); 237 list_move_tail(&qdesc->node, &mchan->active); 238 } 239 240 if (!mchan->running) { 241 struct hidma_desc *desc = list_first_entry(&mchan->active, 242 struct hidma_desc, 243 node); 244 mchan->running = desc; 245 } 246 spin_unlock_irqrestore(&mchan->lock, flags); 247 248 /* PM will be released in hidma_callback function. */ 249 status = pm_runtime_get(dmadev->ddev.dev); 250 if (status < 0) 251 tasklet_schedule(&dmadev->task); 252 else 253 hidma_ll_start(dmadev->lldev); 254 } 255 256 static inline bool hidma_txn_is_success(dma_cookie_t cookie, 257 dma_cookie_t last_success, dma_cookie_t last_used) 258 { 259 if (last_success <= last_used) { 260 if ((cookie <= last_success) || (cookie > last_used)) 261 return true; 262 } else { 263 if ((cookie <= last_success) && (cookie > last_used)) 264 return true; 265 } 266 return false; 267 } 268 269 static enum dma_status hidma_tx_status(struct dma_chan *dmach, 270 dma_cookie_t cookie, 271 struct dma_tx_state *txstate) 272 { 273 struct hidma_chan *mchan = to_hidma_chan(dmach); 274 enum dma_status ret; 275 276 ret = dma_cookie_status(dmach, cookie, txstate); 277 if (ret == DMA_COMPLETE) { 278 bool is_success; 279 280 is_success = hidma_txn_is_success(cookie, mchan->last_success, 281 dmach->cookie); 282 return is_success ? ret : DMA_ERROR; 283 } 284 285 if (mchan->paused && (ret == DMA_IN_PROGRESS)) { 286 unsigned long flags; 287 dma_cookie_t runcookie; 288 289 spin_lock_irqsave(&mchan->lock, flags); 290 if (mchan->running) 291 runcookie = mchan->running->desc.cookie; 292 else 293 runcookie = -EINVAL; 294 295 if (runcookie == cookie) 296 ret = DMA_PAUSED; 297 298 spin_unlock_irqrestore(&mchan->lock, flags); 299 } 300 301 return ret; 302 } 303 304 /* 305 * Submit descriptor to hardware. 306 * Lock the PM for each descriptor we are sending. 307 */ 308 static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd) 309 { 310 struct hidma_chan *mchan = to_hidma_chan(txd->chan); 311 struct hidma_dev *dmadev = mchan->dmadev; 312 struct hidma_desc *mdesc; 313 unsigned long irqflags; 314 dma_cookie_t cookie; 315 316 pm_runtime_get_sync(dmadev->ddev.dev); 317 if (!hidma_ll_isenabled(dmadev->lldev)) { 318 pm_runtime_mark_last_busy(dmadev->ddev.dev); 319 pm_runtime_put_autosuspend(dmadev->ddev.dev); 320 return -ENODEV; 321 } 322 pm_runtime_mark_last_busy(dmadev->ddev.dev); 323 pm_runtime_put_autosuspend(dmadev->ddev.dev); 324 325 mdesc = container_of(txd, struct hidma_desc, desc); 326 spin_lock_irqsave(&mchan->lock, irqflags); 327 328 /* Move descriptor to queued */ 329 list_move_tail(&mdesc->node, &mchan->queued); 330 331 /* Update cookie */ 332 cookie = dma_cookie_assign(txd); 333 334 spin_unlock_irqrestore(&mchan->lock, irqflags); 335 336 return cookie; 337 } 338 339 static int hidma_alloc_chan_resources(struct dma_chan *dmach) 340 { 341 struct hidma_chan *mchan = to_hidma_chan(dmach); 342 struct hidma_dev *dmadev = mchan->dmadev; 343 struct hidma_desc *mdesc, *tmp; 344 unsigned long irqflags; 345 LIST_HEAD(descs); 346 unsigned int i; 347 int rc = 0; 348 349 if (mchan->allocated) 350 return 0; 351 352 /* Alloc descriptors for this channel */ 353 for (i = 0; i < dmadev->nr_descriptors; i++) { 354 mdesc = kzalloc_obj(struct hidma_desc, GFP_NOWAIT); 355 if (!mdesc) { 356 rc = -ENOMEM; 357 break; 358 } 359 dma_async_tx_descriptor_init(&mdesc->desc, dmach); 360 mdesc->desc.tx_submit = hidma_tx_submit; 361 362 rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, 363 "DMA engine", hidma_callback, mdesc, 364 &mdesc->tre_ch); 365 if (rc) { 366 dev_err(dmach->device->dev, 367 "channel alloc failed at %u\n", i); 368 kfree(mdesc); 369 break; 370 } 371 list_add_tail(&mdesc->node, &descs); 372 } 373 374 if (rc) { 375 /* return the allocated descriptors */ 376 list_for_each_entry_safe(mdesc, tmp, &descs, node) { 377 hidma_ll_free(dmadev->lldev, mdesc->tre_ch); 378 kfree(mdesc); 379 } 380 return rc; 381 } 382 383 spin_lock_irqsave(&mchan->lock, irqflags); 384 list_splice_tail_init(&descs, &mchan->free); 385 mchan->allocated = true; 386 spin_unlock_irqrestore(&mchan->lock, irqflags); 387 return 1; 388 } 389 390 static struct dma_async_tx_descriptor * 391 hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src, 392 size_t len, unsigned long flags) 393 { 394 struct hidma_chan *mchan = to_hidma_chan(dmach); 395 struct hidma_desc *mdesc = NULL; 396 struct hidma_dev *mdma = mchan->dmadev; 397 unsigned long irqflags; 398 399 /* Get free descriptor */ 400 spin_lock_irqsave(&mchan->lock, irqflags); 401 if (!list_empty(&mchan->free)) { 402 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); 403 list_del(&mdesc->node); 404 } 405 spin_unlock_irqrestore(&mchan->lock, irqflags); 406 407 if (!mdesc) 408 return NULL; 409 410 mdesc->desc.flags = flags; 411 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, 412 src, dest, len, flags, 413 HIDMA_TRE_MEMCPY); 414 415 /* Place descriptor in prepared list */ 416 spin_lock_irqsave(&mchan->lock, irqflags); 417 list_add_tail(&mdesc->node, &mchan->prepared); 418 spin_unlock_irqrestore(&mchan->lock, irqflags); 419 420 return &mdesc->desc; 421 } 422 423 static struct dma_async_tx_descriptor * 424 hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value, 425 size_t len, unsigned long flags) 426 { 427 struct hidma_chan *mchan = to_hidma_chan(dmach); 428 struct hidma_desc *mdesc = NULL; 429 struct hidma_dev *mdma = mchan->dmadev; 430 unsigned long irqflags; 431 u64 byte_pattern, fill_pattern; 432 433 /* Get free descriptor */ 434 spin_lock_irqsave(&mchan->lock, irqflags); 435 if (!list_empty(&mchan->free)) { 436 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); 437 list_del(&mdesc->node); 438 } 439 spin_unlock_irqrestore(&mchan->lock, irqflags); 440 441 if (!mdesc) 442 return NULL; 443 444 byte_pattern = (char)value; 445 fill_pattern = (byte_pattern << 56) | 446 (byte_pattern << 48) | 447 (byte_pattern << 40) | 448 (byte_pattern << 32) | 449 (byte_pattern << 24) | 450 (byte_pattern << 16) | 451 (byte_pattern << 8) | 452 byte_pattern; 453 454 mdesc->desc.flags = flags; 455 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, 456 fill_pattern, dest, len, flags, 457 HIDMA_TRE_MEMSET); 458 459 /* Place descriptor in prepared list */ 460 spin_lock_irqsave(&mchan->lock, irqflags); 461 list_add_tail(&mdesc->node, &mchan->prepared); 462 spin_unlock_irqrestore(&mchan->lock, irqflags); 463 464 return &mdesc->desc; 465 } 466 467 static int hidma_terminate_channel(struct dma_chan *chan) 468 { 469 struct hidma_chan *mchan = to_hidma_chan(chan); 470 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); 471 struct hidma_desc *tmp, *mdesc; 472 unsigned long irqflags; 473 LIST_HEAD(list); 474 int rc; 475 476 pm_runtime_get_sync(dmadev->ddev.dev); 477 /* give completed requests a chance to finish */ 478 hidma_process_completed(mchan); 479 480 spin_lock_irqsave(&mchan->lock, irqflags); 481 mchan->last_success = 0; 482 list_splice_init(&mchan->active, &list); 483 list_splice_init(&mchan->prepared, &list); 484 list_splice_init(&mchan->completed, &list); 485 list_splice_init(&mchan->queued, &list); 486 spin_unlock_irqrestore(&mchan->lock, irqflags); 487 488 /* this suspends the existing transfer */ 489 rc = hidma_ll_disable(dmadev->lldev); 490 if (rc) { 491 dev_err(dmadev->ddev.dev, "channel did not pause\n"); 492 goto out; 493 } 494 495 /* return all user requests */ 496 list_for_each_entry_safe(mdesc, tmp, &list, node) { 497 struct dma_async_tx_descriptor *txd = &mdesc->desc; 498 499 dma_descriptor_unmap(txd); 500 dmaengine_desc_get_callback_invoke(txd, NULL); 501 dma_run_dependencies(txd); 502 503 /* move myself to free_list */ 504 list_move(&mdesc->node, &mchan->free); 505 } 506 507 rc = hidma_ll_enable(dmadev->lldev); 508 out: 509 pm_runtime_mark_last_busy(dmadev->ddev.dev); 510 pm_runtime_put_autosuspend(dmadev->ddev.dev); 511 return rc; 512 } 513 514 static int hidma_terminate_all(struct dma_chan *chan) 515 { 516 struct hidma_chan *mchan = to_hidma_chan(chan); 517 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); 518 int rc; 519 520 rc = hidma_terminate_channel(chan); 521 if (rc) 522 return rc; 523 524 /* reinitialize the hardware */ 525 pm_runtime_get_sync(dmadev->ddev.dev); 526 rc = hidma_ll_setup(dmadev->lldev); 527 pm_runtime_mark_last_busy(dmadev->ddev.dev); 528 pm_runtime_put_autosuspend(dmadev->ddev.dev); 529 return rc; 530 } 531 532 static void hidma_free_chan_resources(struct dma_chan *dmach) 533 { 534 struct hidma_chan *mchan = to_hidma_chan(dmach); 535 struct hidma_dev *mdma = mchan->dmadev; 536 struct hidma_desc *mdesc, *tmp; 537 unsigned long irqflags; 538 LIST_HEAD(descs); 539 540 /* terminate running transactions and free descriptors */ 541 hidma_terminate_channel(dmach); 542 543 spin_lock_irqsave(&mchan->lock, irqflags); 544 545 /* Move data */ 546 list_splice_tail_init(&mchan->free, &descs); 547 548 /* Free descriptors */ 549 list_for_each_entry_safe(mdesc, tmp, &descs, node) { 550 hidma_ll_free(mdma->lldev, mdesc->tre_ch); 551 list_del(&mdesc->node); 552 kfree(mdesc); 553 } 554 555 mchan->allocated = false; 556 spin_unlock_irqrestore(&mchan->lock, irqflags); 557 } 558 559 static int hidma_pause(struct dma_chan *chan) 560 { 561 struct hidma_chan *mchan; 562 struct hidma_dev *dmadev; 563 564 mchan = to_hidma_chan(chan); 565 dmadev = to_hidma_dev(mchan->chan.device); 566 if (!mchan->paused) { 567 pm_runtime_get_sync(dmadev->ddev.dev); 568 if (hidma_ll_disable(dmadev->lldev)) 569 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); 570 mchan->paused = true; 571 pm_runtime_mark_last_busy(dmadev->ddev.dev); 572 pm_runtime_put_autosuspend(dmadev->ddev.dev); 573 } 574 return 0; 575 } 576 577 static int hidma_resume(struct dma_chan *chan) 578 { 579 struct hidma_chan *mchan; 580 struct hidma_dev *dmadev; 581 int rc = 0; 582 583 mchan = to_hidma_chan(chan); 584 dmadev = to_hidma_dev(mchan->chan.device); 585 if (mchan->paused) { 586 pm_runtime_get_sync(dmadev->ddev.dev); 587 rc = hidma_ll_enable(dmadev->lldev); 588 if (!rc) 589 mchan->paused = false; 590 else 591 dev_err(dmadev->ddev.dev, 592 "failed to resume the channel"); 593 pm_runtime_mark_last_busy(dmadev->ddev.dev); 594 pm_runtime_put_autosuspend(dmadev->ddev.dev); 595 } 596 return rc; 597 } 598 599 static irqreturn_t hidma_chirq_handler(int chirq, void *arg) 600 { 601 struct hidma_lldev *lldev = arg; 602 603 /* 604 * All interrupts are request driven. 605 * HW doesn't send an interrupt by itself. 606 */ 607 return hidma_ll_inthandler(chirq, lldev); 608 } 609 610 #ifdef CONFIG_GENERIC_MSI_IRQ 611 static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg) 612 { 613 struct hidma_lldev **lldevp = arg; 614 struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp); 615 616 return hidma_ll_inthandler_msi(chirq, *lldevp, 617 1 << (chirq - dmadev->msi_virqbase)); 618 } 619 #endif 620 621 static ssize_t hidma_show_values(struct device *dev, 622 struct device_attribute *attr, char *buf) 623 { 624 struct hidma_dev *mdev = dev_get_drvdata(dev); 625 626 if (strcmp(attr->attr.name, "chid") == 0) 627 return sysfs_emit(buf, "%d\n", mdev->chidx); 628 629 return 0; 630 } 631 632 static inline void hidma_sysfs_uninit(struct hidma_dev *dev) 633 { 634 device_remove_file(dev->ddev.dev, dev->chid_attrs); 635 } 636 637 static struct device_attribute* 638 hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode) 639 { 640 struct device_attribute *attrs; 641 char *name_copy; 642 643 attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), 644 GFP_KERNEL); 645 if (!attrs) 646 return NULL; 647 648 name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); 649 if (!name_copy) 650 return NULL; 651 652 attrs->attr.name = name_copy; 653 attrs->attr.mode = mode; 654 attrs->show = hidma_show_values; 655 sysfs_attr_init(&attrs->attr); 656 657 return attrs; 658 } 659 660 static int hidma_sysfs_init(struct hidma_dev *dev) 661 { 662 dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO); 663 if (!dev->chid_attrs) 664 return -ENOMEM; 665 666 return device_create_file(dev->ddev.dev, dev->chid_attrs); 667 } 668 669 #ifdef CONFIG_GENERIC_MSI_IRQ 670 static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 671 { 672 struct device *dev = msi_desc_to_dev(desc); 673 struct hidma_dev *dmadev = dev_get_drvdata(dev); 674 675 if (!desc->msi_index) { 676 writel(msg->address_lo, dmadev->dev_evca + 0x118); 677 writel(msg->address_hi, dmadev->dev_evca + 0x11C); 678 writel(msg->data, dmadev->dev_evca + 0x120); 679 } 680 } 681 #endif 682 683 static void hidma_free_msis(struct hidma_dev *dmadev) 684 { 685 #ifdef CONFIG_GENERIC_MSI_IRQ 686 struct device *dev = dmadev->ddev.dev; 687 int i, virq; 688 689 for (i = 0; i < HIDMA_MSI_INTS; i++) { 690 virq = msi_get_virq(dev, i); 691 if (virq) 692 devm_free_irq(dev, virq, &dmadev->lldev); 693 } 694 695 platform_device_msi_free_irqs_all(dev); 696 #endif 697 } 698 699 static int hidma_request_msi(struct hidma_dev *dmadev, 700 struct platform_device *pdev) 701 { 702 #ifdef CONFIG_GENERIC_MSI_IRQ 703 int rc, i, virq; 704 705 rc = platform_device_msi_init_and_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, 706 hidma_write_msi_msg); 707 if (rc) 708 return rc; 709 710 for (i = 0; i < HIDMA_MSI_INTS; i++) { 711 virq = msi_get_virq(&pdev->dev, i); 712 rc = devm_request_irq(&pdev->dev, virq, 713 hidma_chirq_handler_msi, 714 0, "qcom-hidma-msi", 715 &dmadev->lldev); 716 if (rc) 717 break; 718 if (!i) 719 dmadev->msi_virqbase = virq; 720 } 721 722 if (rc) { 723 /* free allocated MSI interrupts above */ 724 for (--i; i >= 0; i--) { 725 virq = msi_get_virq(&pdev->dev, i); 726 devm_free_irq(&pdev->dev, virq, &dmadev->lldev); 727 } 728 dev_warn(&pdev->dev, 729 "failed to request MSI irq, falling back to wired IRQ\n"); 730 } else { 731 /* Add callback to free MSIs on teardown */ 732 hidma_ll_setup_irq(dmadev->lldev, true); 733 } 734 return rc; 735 #else 736 return -EINVAL; 737 #endif 738 } 739 740 static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap) 741 { 742 enum hidma_cap cap; 743 744 cap = (uintptr_t) device_get_match_data(dev); 745 return cap ? ((cap & test_cap) > 0) : 0; 746 } 747 748 static int hidma_probe(struct platform_device *pdev) 749 { 750 struct hidma_dev *dmadev; 751 struct resource *trca_resource; 752 struct resource *evca_resource; 753 int chirq; 754 void __iomem *evca; 755 void __iomem *trca; 756 int rc; 757 bool msi; 758 759 pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); 760 pm_runtime_use_autosuspend(&pdev->dev); 761 pm_runtime_set_active(&pdev->dev); 762 pm_runtime_enable(&pdev->dev); 763 764 trca = devm_platform_get_and_ioremap_resource(pdev, 0, &trca_resource); 765 if (IS_ERR(trca)) { 766 rc = PTR_ERR(trca); 767 goto bailout; 768 } 769 770 evca = devm_platform_get_and_ioremap_resource(pdev, 1, &evca_resource); 771 if (IS_ERR(evca)) { 772 rc = PTR_ERR(evca); 773 goto bailout; 774 } 775 776 /* 777 * This driver only handles the channel IRQs. 778 * Common IRQ is handled by the management driver. 779 */ 780 chirq = platform_get_irq(pdev, 0); 781 if (chirq < 0) { 782 rc = chirq; 783 goto bailout; 784 } 785 786 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); 787 if (!dmadev) { 788 rc = -ENOMEM; 789 goto bailout; 790 } 791 792 INIT_LIST_HEAD(&dmadev->ddev.channels); 793 spin_lock_init(&dmadev->lock); 794 dmadev->ddev.dev = &pdev->dev; 795 pm_runtime_get_sync(dmadev->ddev.dev); 796 797 dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); 798 dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask); 799 if (WARN_ON(!pdev->dev.dma_mask)) { 800 rc = -ENXIO; 801 goto dmafree; 802 } 803 804 dmadev->dev_evca = evca; 805 dmadev->evca_resource = evca_resource; 806 dmadev->dev_trca = trca; 807 dmadev->trca_resource = trca_resource; 808 dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; 809 dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset; 810 dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; 811 dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; 812 dmadev->ddev.device_tx_status = hidma_tx_status; 813 dmadev->ddev.device_issue_pending = hidma_issue_pending; 814 dmadev->ddev.device_pause = hidma_pause; 815 dmadev->ddev.device_resume = hidma_resume; 816 dmadev->ddev.device_terminate_all = hidma_terminate_all; 817 dmadev->ddev.copy_align = 8; 818 819 /* 820 * Determine the MSI capability of the platform. Old HW doesn't 821 * support MSI. 822 */ 823 msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP); 824 device_property_read_u32(&pdev->dev, "desc-count", 825 &dmadev->nr_descriptors); 826 827 if (nr_desc_prm) { 828 dev_info(&pdev->dev, "overriding number of descriptors as %d\n", 829 nr_desc_prm); 830 dmadev->nr_descriptors = nr_desc_prm; 831 } 832 833 if (!dmadev->nr_descriptors) 834 dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; 835 836 if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP)) 837 dmadev->chidx = readl(dmadev->dev_trca + 0x40); 838 else 839 dmadev->chidx = readl(dmadev->dev_trca + 0x28); 840 841 /* Set DMA mask to 64 bits. */ 842 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 843 if (rc) { 844 dev_warn(&pdev->dev, "unable to set coherent mask to 64"); 845 goto dmafree; 846 } 847 848 dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, 849 dmadev->nr_descriptors, dmadev->dev_trca, 850 dmadev->dev_evca, dmadev->chidx); 851 if (!dmadev->lldev) { 852 rc = -EPROBE_DEFER; 853 goto dmafree; 854 } 855 856 platform_set_drvdata(pdev, dmadev); 857 if (msi) 858 rc = hidma_request_msi(dmadev, pdev); 859 860 if (!msi || rc) { 861 hidma_ll_setup_irq(dmadev->lldev, false); 862 rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, 863 0, "qcom-hidma", dmadev->lldev); 864 if (rc) 865 goto uninit; 866 } 867 868 INIT_LIST_HEAD(&dmadev->ddev.channels); 869 rc = hidma_chan_init(dmadev, 0); 870 if (rc) 871 goto uninit; 872 873 rc = dma_async_device_register(&dmadev->ddev); 874 if (rc) 875 goto uninit; 876 877 dmadev->irq = chirq; 878 tasklet_setup(&dmadev->task, hidma_issue_task); 879 hidma_debug_init(dmadev); 880 hidma_sysfs_init(dmadev); 881 dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); 882 pm_runtime_mark_last_busy(dmadev->ddev.dev); 883 pm_runtime_put_autosuspend(dmadev->ddev.dev); 884 return 0; 885 886 uninit: 887 if (msi) 888 hidma_free_msis(dmadev); 889 890 hidma_ll_uninit(dmadev->lldev); 891 dmafree: 892 if (dmadev) 893 hidma_free(dmadev); 894 bailout: 895 pm_runtime_put_sync(&pdev->dev); 896 pm_runtime_disable(&pdev->dev); 897 return rc; 898 } 899 900 static void hidma_shutdown(struct platform_device *pdev) 901 { 902 struct hidma_dev *dmadev = platform_get_drvdata(pdev); 903 904 dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n"); 905 906 pm_runtime_get_sync(dmadev->ddev.dev); 907 if (hidma_ll_disable(dmadev->lldev)) 908 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); 909 pm_runtime_mark_last_busy(dmadev->ddev.dev); 910 pm_runtime_put_autosuspend(dmadev->ddev.dev); 911 912 } 913 914 static void hidma_remove(struct platform_device *pdev) 915 { 916 struct hidma_dev *dmadev = platform_get_drvdata(pdev); 917 918 pm_runtime_get_sync(dmadev->ddev.dev); 919 dma_async_device_unregister(&dmadev->ddev); 920 if (!dmadev->lldev->msi_support) 921 devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); 922 else 923 hidma_free_msis(dmadev); 924 925 tasklet_kill(&dmadev->task); 926 hidma_sysfs_uninit(dmadev); 927 hidma_debug_uninit(dmadev); 928 hidma_ll_uninit(dmadev->lldev); 929 hidma_free(dmadev); 930 931 dev_info(&pdev->dev, "HI-DMA engine removed\n"); 932 pm_runtime_put_sync_suspend(&pdev->dev); 933 pm_runtime_disable(&pdev->dev); 934 } 935 936 #if IS_ENABLED(CONFIG_ACPI) 937 static const struct acpi_device_id hidma_acpi_ids[] = { 938 {"QCOM8061"}, 939 {"QCOM8062", HIDMA_MSI_CAP}, 940 {"QCOM8063", (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP)}, 941 {}, 942 }; 943 MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); 944 #endif 945 946 static struct platform_driver hidma_driver = { 947 .probe = hidma_probe, 948 .remove = hidma_remove, 949 .shutdown = hidma_shutdown, 950 .driver = { 951 .name = "hidma", 952 .acpi_match_table = ACPI_PTR(hidma_acpi_ids), 953 }, 954 }; 955 956 module_platform_driver(hidma_driver); 957 MODULE_DESCRIPTION("Qualcomm Technologies HIDMA Channel support"); 958 MODULE_LICENSE("GPL v2"); 959