167a2003eSSinan Kaya /* 267a2003eSSinan Kaya * Qualcomm Technologies HIDMA DMA engine interface 367a2003eSSinan Kaya * 413058e33SSinan Kaya * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved. 567a2003eSSinan Kaya * 667a2003eSSinan Kaya * This program is free software; you can redistribute it and/or modify 767a2003eSSinan Kaya * it under the terms of the GNU General Public License version 2 and 867a2003eSSinan Kaya * only version 2 as published by the Free Software Foundation. 967a2003eSSinan Kaya * 1067a2003eSSinan Kaya * This program is distributed in the hope that it will be useful, 1167a2003eSSinan Kaya * but WITHOUT ANY WARRANTY; without even the implied warranty of 1267a2003eSSinan Kaya * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1367a2003eSSinan Kaya * GNU General Public License for more details. 1467a2003eSSinan Kaya */ 1567a2003eSSinan Kaya 1667a2003eSSinan Kaya /* 1767a2003eSSinan Kaya * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008. 1867a2003eSSinan Kaya * Copyright (C) Semihalf 2009 1967a2003eSSinan Kaya * Copyright (C) Ilya Yanok, Emcraft Systems 2010 2067a2003eSSinan Kaya * Copyright (C) Alexander Popov, Promcontroller 2014 2167a2003eSSinan Kaya * 2267a2003eSSinan Kaya * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description 2367a2003eSSinan Kaya * (defines, structures and comments) was taken from MPC5121 DMA driver 2467a2003eSSinan Kaya * written by Hongjun Chen <hong-jun.chen@freescale.com>. 2567a2003eSSinan Kaya * 2667a2003eSSinan Kaya * Approved as OSADL project by a majority of OSADL members and funded 2767a2003eSSinan Kaya * by OSADL membership fees in 2009; for details see www.osadl.org. 2867a2003eSSinan Kaya * 2967a2003eSSinan Kaya * This program is free software; you can redistribute it and/or modify it 3067a2003eSSinan Kaya * under the terms of the GNU General Public License as published by the Free 3167a2003eSSinan Kaya * Software Foundation; either version 2 of the License, or (at your option) 3267a2003eSSinan Kaya * any later version. 3367a2003eSSinan Kaya * 3467a2003eSSinan Kaya * This program is distributed in the hope that it will be useful, but WITHOUT 3567a2003eSSinan Kaya * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 3667a2003eSSinan Kaya * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 3767a2003eSSinan Kaya * more details. 3867a2003eSSinan Kaya * 3967a2003eSSinan Kaya * The full GNU General Public License is included in this distribution in the 4067a2003eSSinan Kaya * file called COPYING. 4167a2003eSSinan Kaya */ 4267a2003eSSinan Kaya 4367a2003eSSinan Kaya /* Linux Foundation elects GPLv2 license only. */ 4467a2003eSSinan Kaya 4567a2003eSSinan Kaya #include <linux/dmaengine.h> 4667a2003eSSinan Kaya #include <linux/dma-mapping.h> 4767a2003eSSinan Kaya #include <linux/list.h> 4867a2003eSSinan Kaya #include <linux/module.h> 4967a2003eSSinan Kaya #include <linux/platform_device.h> 5067a2003eSSinan Kaya #include <linux/slab.h> 5167a2003eSSinan Kaya #include <linux/spinlock.h> 5267a2003eSSinan Kaya #include <linux/of_dma.h> 5367a2003eSSinan Kaya #include <linux/property.h> 5467a2003eSSinan Kaya #include <linux/delay.h> 5567a2003eSSinan Kaya #include <linux/acpi.h> 5667a2003eSSinan Kaya #include <linux/irq.h> 5767a2003eSSinan Kaya #include <linux/atomic.h> 5867a2003eSSinan Kaya #include <linux/pm_runtime.h> 591c0e3e82SSinan Kaya #include <linux/msi.h> 6067a2003eSSinan Kaya 6167a2003eSSinan Kaya #include "../dmaengine.h" 6267a2003eSSinan Kaya #include "hidma.h" 6367a2003eSSinan Kaya 6467a2003eSSinan Kaya /* 6567a2003eSSinan Kaya * Default idle time is 2 seconds. This parameter can 6667a2003eSSinan Kaya * be overridden by changing the following 6767a2003eSSinan Kaya * /sys/bus/platform/devices/QCOM8061:<xy>/power/autosuspend_delay_ms 6867a2003eSSinan Kaya * during kernel boot. 6967a2003eSSinan Kaya */ 7067a2003eSSinan Kaya #define HIDMA_AUTOSUSPEND_TIMEOUT 2000 7167a2003eSSinan Kaya #define HIDMA_ERR_INFO_SW 0xFF 7267a2003eSSinan Kaya #define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0 7367a2003eSSinan Kaya #define HIDMA_NR_DEFAULT_DESC 10 741c0e3e82SSinan Kaya #define HIDMA_MSI_INTS 11 7567a2003eSSinan Kaya 7667a2003eSSinan Kaya static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev) 7767a2003eSSinan Kaya { 7867a2003eSSinan Kaya return container_of(dmadev, struct hidma_dev, ddev); 7967a2003eSSinan Kaya } 8067a2003eSSinan Kaya 8167a2003eSSinan Kaya static inline 8267a2003eSSinan Kaya struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp) 8367a2003eSSinan Kaya { 8467a2003eSSinan Kaya return container_of(_lldevp, struct hidma_dev, lldev); 8567a2003eSSinan Kaya } 8667a2003eSSinan Kaya 8767a2003eSSinan Kaya static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach) 8867a2003eSSinan Kaya { 8967a2003eSSinan Kaya return container_of(dmach, struct hidma_chan, chan); 9067a2003eSSinan Kaya } 9167a2003eSSinan Kaya 9267a2003eSSinan Kaya static inline 9367a2003eSSinan Kaya struct hidma_desc *to_hidma_desc(struct dma_async_tx_descriptor *t) 9467a2003eSSinan Kaya { 9567a2003eSSinan Kaya return container_of(t, struct hidma_desc, desc); 9667a2003eSSinan Kaya } 9767a2003eSSinan Kaya 9867a2003eSSinan Kaya static void hidma_free(struct hidma_dev *dmadev) 9967a2003eSSinan Kaya { 10067a2003eSSinan Kaya INIT_LIST_HEAD(&dmadev->ddev.channels); 10167a2003eSSinan Kaya } 10267a2003eSSinan Kaya 10367a2003eSSinan Kaya static unsigned int nr_desc_prm; 10467a2003eSSinan Kaya module_param(nr_desc_prm, uint, 0644); 10567a2003eSSinan Kaya MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)"); 10667a2003eSSinan Kaya 10767a2003eSSinan Kaya 10867a2003eSSinan Kaya /* process completed descriptors */ 10967a2003eSSinan Kaya static void hidma_process_completed(struct hidma_chan *mchan) 11067a2003eSSinan Kaya { 11167a2003eSSinan Kaya struct dma_device *ddev = mchan->chan.device; 11267a2003eSSinan Kaya struct hidma_dev *mdma = to_hidma_dev(ddev); 11367a2003eSSinan Kaya struct dma_async_tx_descriptor *desc; 11467a2003eSSinan Kaya dma_cookie_t last_cookie; 11567a2003eSSinan Kaya struct hidma_desc *mdesc; 1168a31f8b5SSinan Kaya struct hidma_desc *next; 11767a2003eSSinan Kaya unsigned long irqflags; 11867a2003eSSinan Kaya struct list_head list; 11967a2003eSSinan Kaya 12067a2003eSSinan Kaya INIT_LIST_HEAD(&list); 12167a2003eSSinan Kaya 12267a2003eSSinan Kaya /* Get all completed descriptors */ 12367a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 12467a2003eSSinan Kaya list_splice_tail_init(&mchan->completed, &list); 12567a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 12667a2003eSSinan Kaya 12767a2003eSSinan Kaya /* Execute callbacks and run dependencies */ 1288a31f8b5SSinan Kaya list_for_each_entry_safe(mdesc, next, &list, node) { 12967a2003eSSinan Kaya enum dma_status llstat; 1308a31f8b5SSinan Kaya struct dmaengine_desc_callback cb; 13155c370e5SSinan Kaya struct dmaengine_result result; 13267a2003eSSinan Kaya 13367a2003eSSinan Kaya desc = &mdesc->desc; 134793ae66cSSinan Kaya last_cookie = desc->cookie; 13567a2003eSSinan Kaya 13667a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 13767a2003eSSinan Kaya dma_cookie_complete(desc); 13867a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 13967a2003eSSinan Kaya 14067a2003eSSinan Kaya llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); 1418a31f8b5SSinan Kaya dmaengine_desc_get_callback(desc, &cb); 14267a2003eSSinan Kaya 14367a2003eSSinan Kaya dma_run_dependencies(desc); 14467a2003eSSinan Kaya 14567a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 1468a31f8b5SSinan Kaya list_move(&mdesc->node, &mchan->free); 14767a2003eSSinan Kaya 148793ae66cSSinan Kaya if (llstat == DMA_COMPLETE) { 149793ae66cSSinan Kaya mchan->last_success = last_cookie; 15055c370e5SSinan Kaya result.result = DMA_TRANS_NOERROR; 151793ae66cSSinan Kaya } else 15255c370e5SSinan Kaya result.result = DMA_TRANS_ABORTED; 15355c370e5SSinan Kaya 15455c370e5SSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 15555c370e5SSinan Kaya 15655c370e5SSinan Kaya dmaengine_desc_callback_invoke(&cb, &result); 1578a31f8b5SSinan Kaya } 15867a2003eSSinan Kaya } 15967a2003eSSinan Kaya 16067a2003eSSinan Kaya /* 16167a2003eSSinan Kaya * Called once for each submitted descriptor. 16267a2003eSSinan Kaya * PM is locked once for each descriptor that is currently 16367a2003eSSinan Kaya * in execution. 16467a2003eSSinan Kaya */ 16567a2003eSSinan Kaya static void hidma_callback(void *data) 16667a2003eSSinan Kaya { 16767a2003eSSinan Kaya struct hidma_desc *mdesc = data; 16867a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); 16967a2003eSSinan Kaya struct dma_device *ddev = mchan->chan.device; 17067a2003eSSinan Kaya struct hidma_dev *dmadev = to_hidma_dev(ddev); 17167a2003eSSinan Kaya unsigned long irqflags; 17267a2003eSSinan Kaya bool queued = false; 17367a2003eSSinan Kaya 17467a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 17567a2003eSSinan Kaya if (mdesc->node.next) { 17667a2003eSSinan Kaya /* Delete from the active list, add to completed list */ 17767a2003eSSinan Kaya list_move_tail(&mdesc->node, &mchan->completed); 17867a2003eSSinan Kaya queued = true; 17967a2003eSSinan Kaya 18067a2003eSSinan Kaya /* calculate the next running descriptor */ 18167a2003eSSinan Kaya mchan->running = list_first_entry(&mchan->active, 18267a2003eSSinan Kaya struct hidma_desc, node); 18367a2003eSSinan Kaya } 18467a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 18567a2003eSSinan Kaya 18667a2003eSSinan Kaya hidma_process_completed(mchan); 18767a2003eSSinan Kaya 18867a2003eSSinan Kaya if (queued) { 18967a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 19067a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 19167a2003eSSinan Kaya } 19267a2003eSSinan Kaya } 19367a2003eSSinan Kaya 19467a2003eSSinan Kaya static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig) 19567a2003eSSinan Kaya { 19667a2003eSSinan Kaya struct hidma_chan *mchan; 19767a2003eSSinan Kaya struct dma_device *ddev; 19867a2003eSSinan Kaya 19967a2003eSSinan Kaya mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); 20067a2003eSSinan Kaya if (!mchan) 20167a2003eSSinan Kaya return -ENOMEM; 20267a2003eSSinan Kaya 20367a2003eSSinan Kaya ddev = &dmadev->ddev; 20467a2003eSSinan Kaya mchan->dma_sig = dma_sig; 20567a2003eSSinan Kaya mchan->dmadev = dmadev; 20667a2003eSSinan Kaya mchan->chan.device = ddev; 20767a2003eSSinan Kaya dma_cookie_init(&mchan->chan); 20867a2003eSSinan Kaya 20967a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->free); 21067a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->prepared); 21167a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->active); 21267a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->completed); 213*99efdb3eSSinan Kaya INIT_LIST_HEAD(&mchan->queued); 21467a2003eSSinan Kaya 21567a2003eSSinan Kaya spin_lock_init(&mchan->lock); 21667a2003eSSinan Kaya list_add_tail(&mchan->chan.device_node, &ddev->channels); 21767a2003eSSinan Kaya dmadev->ddev.chancnt++; 21867a2003eSSinan Kaya return 0; 21967a2003eSSinan Kaya } 22067a2003eSSinan Kaya 22167a2003eSSinan Kaya static void hidma_issue_task(unsigned long arg) 22267a2003eSSinan Kaya { 22367a2003eSSinan Kaya struct hidma_dev *dmadev = (struct hidma_dev *)arg; 22467a2003eSSinan Kaya 22567a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 22667a2003eSSinan Kaya hidma_ll_start(dmadev->lldev); 22767a2003eSSinan Kaya } 22867a2003eSSinan Kaya 22967a2003eSSinan Kaya static void hidma_issue_pending(struct dma_chan *dmach) 23067a2003eSSinan Kaya { 23167a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 23267a2003eSSinan Kaya struct hidma_dev *dmadev = mchan->dmadev; 23367a2003eSSinan Kaya unsigned long flags; 234*99efdb3eSSinan Kaya struct hidma_desc *qdesc, *next; 23567a2003eSSinan Kaya int status; 23667a2003eSSinan Kaya 23767a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, flags); 238*99efdb3eSSinan Kaya list_for_each_entry_safe(qdesc, next, &mchan->queued, node) { 239*99efdb3eSSinan Kaya hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch); 240*99efdb3eSSinan Kaya list_move_tail(&qdesc->node, &mchan->active); 241*99efdb3eSSinan Kaya } 242*99efdb3eSSinan Kaya 24367a2003eSSinan Kaya if (!mchan->running) { 24467a2003eSSinan Kaya struct hidma_desc *desc = list_first_entry(&mchan->active, 24567a2003eSSinan Kaya struct hidma_desc, 24667a2003eSSinan Kaya node); 24767a2003eSSinan Kaya mchan->running = desc; 24867a2003eSSinan Kaya } 24967a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, flags); 25067a2003eSSinan Kaya 25167a2003eSSinan Kaya /* PM will be released in hidma_callback function. */ 25267a2003eSSinan Kaya status = pm_runtime_get(dmadev->ddev.dev); 25367a2003eSSinan Kaya if (status < 0) 25467a2003eSSinan Kaya tasklet_schedule(&dmadev->task); 25567a2003eSSinan Kaya else 25667a2003eSSinan Kaya hidma_ll_start(dmadev->lldev); 25767a2003eSSinan Kaya } 25867a2003eSSinan Kaya 259793ae66cSSinan Kaya static inline bool hidma_txn_is_success(dma_cookie_t cookie, 260793ae66cSSinan Kaya dma_cookie_t last_success, dma_cookie_t last_used) 261793ae66cSSinan Kaya { 262793ae66cSSinan Kaya if (last_success <= last_used) { 263793ae66cSSinan Kaya if ((cookie <= last_success) || (cookie > last_used)) 264793ae66cSSinan Kaya return true; 265793ae66cSSinan Kaya } else { 266793ae66cSSinan Kaya if ((cookie <= last_success) && (cookie > last_used)) 267793ae66cSSinan Kaya return true; 268793ae66cSSinan Kaya } 269793ae66cSSinan Kaya return false; 270793ae66cSSinan Kaya } 271793ae66cSSinan Kaya 27267a2003eSSinan Kaya static enum dma_status hidma_tx_status(struct dma_chan *dmach, 27367a2003eSSinan Kaya dma_cookie_t cookie, 27467a2003eSSinan Kaya struct dma_tx_state *txstate) 27567a2003eSSinan Kaya { 27667a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 27767a2003eSSinan Kaya enum dma_status ret; 27867a2003eSSinan Kaya 27967a2003eSSinan Kaya ret = dma_cookie_status(dmach, cookie, txstate); 280793ae66cSSinan Kaya if (ret == DMA_COMPLETE) { 281793ae66cSSinan Kaya bool is_success; 282793ae66cSSinan Kaya 283793ae66cSSinan Kaya is_success = hidma_txn_is_success(cookie, mchan->last_success, 284793ae66cSSinan Kaya dmach->cookie); 285793ae66cSSinan Kaya return is_success ? ret : DMA_ERROR; 286793ae66cSSinan Kaya } 28767a2003eSSinan Kaya 28867a2003eSSinan Kaya if (mchan->paused && (ret == DMA_IN_PROGRESS)) { 28967a2003eSSinan Kaya unsigned long flags; 29067a2003eSSinan Kaya dma_cookie_t runcookie; 29167a2003eSSinan Kaya 29267a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, flags); 29367a2003eSSinan Kaya if (mchan->running) 29467a2003eSSinan Kaya runcookie = mchan->running->desc.cookie; 29567a2003eSSinan Kaya else 29667a2003eSSinan Kaya runcookie = -EINVAL; 29767a2003eSSinan Kaya 29867a2003eSSinan Kaya if (runcookie == cookie) 29967a2003eSSinan Kaya ret = DMA_PAUSED; 30067a2003eSSinan Kaya 30167a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, flags); 30267a2003eSSinan Kaya } 30367a2003eSSinan Kaya 30467a2003eSSinan Kaya return ret; 30567a2003eSSinan Kaya } 30667a2003eSSinan Kaya 30767a2003eSSinan Kaya /* 30867a2003eSSinan Kaya * Submit descriptor to hardware. 30967a2003eSSinan Kaya * Lock the PM for each descriptor we are sending. 31067a2003eSSinan Kaya */ 31167a2003eSSinan Kaya static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd) 31267a2003eSSinan Kaya { 31367a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(txd->chan); 31467a2003eSSinan Kaya struct hidma_dev *dmadev = mchan->dmadev; 31567a2003eSSinan Kaya struct hidma_desc *mdesc; 31667a2003eSSinan Kaya unsigned long irqflags; 31767a2003eSSinan Kaya dma_cookie_t cookie; 31867a2003eSSinan Kaya 31967a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 32067a2003eSSinan Kaya if (!hidma_ll_isenabled(dmadev->lldev)) { 32167a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 32267a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 32367a2003eSSinan Kaya return -ENODEV; 32467a2003eSSinan Kaya } 325*99efdb3eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 326*99efdb3eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 32767a2003eSSinan Kaya 32867a2003eSSinan Kaya mdesc = container_of(txd, struct hidma_desc, desc); 32967a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 33067a2003eSSinan Kaya 331*99efdb3eSSinan Kaya /* Move descriptor to queued */ 332*99efdb3eSSinan Kaya list_move_tail(&mdesc->node, &mchan->queued); 33367a2003eSSinan Kaya 33467a2003eSSinan Kaya /* Update cookie */ 33567a2003eSSinan Kaya cookie = dma_cookie_assign(txd); 33667a2003eSSinan Kaya 33767a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 33867a2003eSSinan Kaya 33967a2003eSSinan Kaya return cookie; 34067a2003eSSinan Kaya } 34167a2003eSSinan Kaya 34267a2003eSSinan Kaya static int hidma_alloc_chan_resources(struct dma_chan *dmach) 34367a2003eSSinan Kaya { 34467a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 34567a2003eSSinan Kaya struct hidma_dev *dmadev = mchan->dmadev; 34667a2003eSSinan Kaya struct hidma_desc *mdesc, *tmp; 34767a2003eSSinan Kaya unsigned long irqflags; 34867a2003eSSinan Kaya LIST_HEAD(descs); 34967a2003eSSinan Kaya unsigned int i; 35067a2003eSSinan Kaya int rc = 0; 35167a2003eSSinan Kaya 35267a2003eSSinan Kaya if (mchan->allocated) 35367a2003eSSinan Kaya return 0; 35467a2003eSSinan Kaya 35567a2003eSSinan Kaya /* Alloc descriptors for this channel */ 35667a2003eSSinan Kaya for (i = 0; i < dmadev->nr_descriptors; i++) { 35767a2003eSSinan Kaya mdesc = kzalloc(sizeof(struct hidma_desc), GFP_NOWAIT); 35867a2003eSSinan Kaya if (!mdesc) { 35967a2003eSSinan Kaya rc = -ENOMEM; 36067a2003eSSinan Kaya break; 36167a2003eSSinan Kaya } 36267a2003eSSinan Kaya dma_async_tx_descriptor_init(&mdesc->desc, dmach); 36367a2003eSSinan Kaya mdesc->desc.tx_submit = hidma_tx_submit; 36467a2003eSSinan Kaya 36567a2003eSSinan Kaya rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, 36667a2003eSSinan Kaya "DMA engine", hidma_callback, mdesc, 36767a2003eSSinan Kaya &mdesc->tre_ch); 36867a2003eSSinan Kaya if (rc) { 36967a2003eSSinan Kaya dev_err(dmach->device->dev, 37067a2003eSSinan Kaya "channel alloc failed at %u\n", i); 37167a2003eSSinan Kaya kfree(mdesc); 37267a2003eSSinan Kaya break; 37367a2003eSSinan Kaya } 37467a2003eSSinan Kaya list_add_tail(&mdesc->node, &descs); 37567a2003eSSinan Kaya } 37667a2003eSSinan Kaya 37767a2003eSSinan Kaya if (rc) { 37867a2003eSSinan Kaya /* return the allocated descriptors */ 37967a2003eSSinan Kaya list_for_each_entry_safe(mdesc, tmp, &descs, node) { 38067a2003eSSinan Kaya hidma_ll_free(dmadev->lldev, mdesc->tre_ch); 38167a2003eSSinan Kaya kfree(mdesc); 38267a2003eSSinan Kaya } 38367a2003eSSinan Kaya return rc; 38467a2003eSSinan Kaya } 38567a2003eSSinan Kaya 38667a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 38767a2003eSSinan Kaya list_splice_tail_init(&descs, &mchan->free); 38867a2003eSSinan Kaya mchan->allocated = true; 38967a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 39067a2003eSSinan Kaya return 1; 39167a2003eSSinan Kaya } 39267a2003eSSinan Kaya 39367a2003eSSinan Kaya static struct dma_async_tx_descriptor * 39467a2003eSSinan Kaya hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src, 39567a2003eSSinan Kaya size_t len, unsigned long flags) 39667a2003eSSinan Kaya { 39767a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 39867a2003eSSinan Kaya struct hidma_desc *mdesc = NULL; 39967a2003eSSinan Kaya struct hidma_dev *mdma = mchan->dmadev; 40067a2003eSSinan Kaya unsigned long irqflags; 40167a2003eSSinan Kaya 40267a2003eSSinan Kaya /* Get free descriptor */ 40367a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 40467a2003eSSinan Kaya if (!list_empty(&mchan->free)) { 40567a2003eSSinan Kaya mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); 40667a2003eSSinan Kaya list_del(&mdesc->node); 40767a2003eSSinan Kaya } 40867a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 40967a2003eSSinan Kaya 41067a2003eSSinan Kaya if (!mdesc) 41167a2003eSSinan Kaya return NULL; 41267a2003eSSinan Kaya 41367a2003eSSinan Kaya hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, 41467a2003eSSinan Kaya src, dest, len, flags); 41567a2003eSSinan Kaya 41667a2003eSSinan Kaya /* Place descriptor in prepared list */ 41767a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 41867a2003eSSinan Kaya list_add_tail(&mdesc->node, &mchan->prepared); 41967a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 42067a2003eSSinan Kaya 42167a2003eSSinan Kaya return &mdesc->desc; 42267a2003eSSinan Kaya } 42367a2003eSSinan Kaya 42467a2003eSSinan Kaya static int hidma_terminate_channel(struct dma_chan *chan) 42567a2003eSSinan Kaya { 42667a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(chan); 42767a2003eSSinan Kaya struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); 42867a2003eSSinan Kaya struct hidma_desc *tmp, *mdesc; 42967a2003eSSinan Kaya unsigned long irqflags; 43067a2003eSSinan Kaya LIST_HEAD(list); 43167a2003eSSinan Kaya int rc; 43267a2003eSSinan Kaya 43367a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 43467a2003eSSinan Kaya /* give completed requests a chance to finish */ 43567a2003eSSinan Kaya hidma_process_completed(mchan); 43667a2003eSSinan Kaya 43767a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 438793ae66cSSinan Kaya mchan->last_success = 0; 43967a2003eSSinan Kaya list_splice_init(&mchan->active, &list); 44067a2003eSSinan Kaya list_splice_init(&mchan->prepared, &list); 44167a2003eSSinan Kaya list_splice_init(&mchan->completed, &list); 442*99efdb3eSSinan Kaya list_splice_init(&mchan->queued, &list); 44367a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 44467a2003eSSinan Kaya 44567a2003eSSinan Kaya /* this suspends the existing transfer */ 446d1615ca2SSinan Kaya rc = hidma_ll_disable(dmadev->lldev); 44767a2003eSSinan Kaya if (rc) { 44867a2003eSSinan Kaya dev_err(dmadev->ddev.dev, "channel did not pause\n"); 44967a2003eSSinan Kaya goto out; 45067a2003eSSinan Kaya } 45167a2003eSSinan Kaya 45267a2003eSSinan Kaya /* return all user requests */ 45367a2003eSSinan Kaya list_for_each_entry_safe(mdesc, tmp, &list, node) { 45467a2003eSSinan Kaya struct dma_async_tx_descriptor *txd = &mdesc->desc; 45567a2003eSSinan Kaya 45667a2003eSSinan Kaya dma_descriptor_unmap(txd); 4575ade6683SDave Jiang dmaengine_desc_get_callback_invoke(txd, NULL); 45867a2003eSSinan Kaya dma_run_dependencies(txd); 45967a2003eSSinan Kaya 46067a2003eSSinan Kaya /* move myself to free_list */ 46167a2003eSSinan Kaya list_move(&mdesc->node, &mchan->free); 46267a2003eSSinan Kaya } 46367a2003eSSinan Kaya 464d1615ca2SSinan Kaya rc = hidma_ll_enable(dmadev->lldev); 46567a2003eSSinan Kaya out: 46667a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 46767a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 46867a2003eSSinan Kaya return rc; 46967a2003eSSinan Kaya } 47067a2003eSSinan Kaya 47167a2003eSSinan Kaya static int hidma_terminate_all(struct dma_chan *chan) 47267a2003eSSinan Kaya { 47367a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(chan); 47467a2003eSSinan Kaya struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); 47567a2003eSSinan Kaya int rc; 47667a2003eSSinan Kaya 47767a2003eSSinan Kaya rc = hidma_terminate_channel(chan); 47867a2003eSSinan Kaya if (rc) 47967a2003eSSinan Kaya return rc; 48067a2003eSSinan Kaya 48167a2003eSSinan Kaya /* reinitialize the hardware */ 48267a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 48367a2003eSSinan Kaya rc = hidma_ll_setup(dmadev->lldev); 48467a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 48567a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 48667a2003eSSinan Kaya return rc; 48767a2003eSSinan Kaya } 48867a2003eSSinan Kaya 48967a2003eSSinan Kaya static void hidma_free_chan_resources(struct dma_chan *dmach) 49067a2003eSSinan Kaya { 49167a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 49267a2003eSSinan Kaya struct hidma_dev *mdma = mchan->dmadev; 49367a2003eSSinan Kaya struct hidma_desc *mdesc, *tmp; 49467a2003eSSinan Kaya unsigned long irqflags; 49567a2003eSSinan Kaya LIST_HEAD(descs); 49667a2003eSSinan Kaya 49767a2003eSSinan Kaya /* terminate running transactions and free descriptors */ 49867a2003eSSinan Kaya hidma_terminate_channel(dmach); 49967a2003eSSinan Kaya 50067a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 50167a2003eSSinan Kaya 50267a2003eSSinan Kaya /* Move data */ 50367a2003eSSinan Kaya list_splice_tail_init(&mchan->free, &descs); 50467a2003eSSinan Kaya 50567a2003eSSinan Kaya /* Free descriptors */ 50667a2003eSSinan Kaya list_for_each_entry_safe(mdesc, tmp, &descs, node) { 50767a2003eSSinan Kaya hidma_ll_free(mdma->lldev, mdesc->tre_ch); 50867a2003eSSinan Kaya list_del(&mdesc->node); 50967a2003eSSinan Kaya kfree(mdesc); 51067a2003eSSinan Kaya } 51167a2003eSSinan Kaya 51267a2003eSSinan Kaya mchan->allocated = 0; 51367a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 51467a2003eSSinan Kaya } 51567a2003eSSinan Kaya 51667a2003eSSinan Kaya static int hidma_pause(struct dma_chan *chan) 51767a2003eSSinan Kaya { 51867a2003eSSinan Kaya struct hidma_chan *mchan; 51967a2003eSSinan Kaya struct hidma_dev *dmadev; 52067a2003eSSinan Kaya 52167a2003eSSinan Kaya mchan = to_hidma_chan(chan); 52267a2003eSSinan Kaya dmadev = to_hidma_dev(mchan->chan.device); 52367a2003eSSinan Kaya if (!mchan->paused) { 52467a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 525d1615ca2SSinan Kaya if (hidma_ll_disable(dmadev->lldev)) 52667a2003eSSinan Kaya dev_warn(dmadev->ddev.dev, "channel did not stop\n"); 52767a2003eSSinan Kaya mchan->paused = true; 52867a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 52967a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 53067a2003eSSinan Kaya } 53167a2003eSSinan Kaya return 0; 53267a2003eSSinan Kaya } 53367a2003eSSinan Kaya 53467a2003eSSinan Kaya static int hidma_resume(struct dma_chan *chan) 53567a2003eSSinan Kaya { 53667a2003eSSinan Kaya struct hidma_chan *mchan; 53767a2003eSSinan Kaya struct hidma_dev *dmadev; 53867a2003eSSinan Kaya int rc = 0; 53967a2003eSSinan Kaya 54067a2003eSSinan Kaya mchan = to_hidma_chan(chan); 54167a2003eSSinan Kaya dmadev = to_hidma_dev(mchan->chan.device); 54267a2003eSSinan Kaya if (mchan->paused) { 54367a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 544d1615ca2SSinan Kaya rc = hidma_ll_enable(dmadev->lldev); 54567a2003eSSinan Kaya if (!rc) 54667a2003eSSinan Kaya mchan->paused = false; 54767a2003eSSinan Kaya else 54867a2003eSSinan Kaya dev_err(dmadev->ddev.dev, 54967a2003eSSinan Kaya "failed to resume the channel"); 55067a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 55167a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 55267a2003eSSinan Kaya } 55367a2003eSSinan Kaya return rc; 55467a2003eSSinan Kaya } 55567a2003eSSinan Kaya 55667a2003eSSinan Kaya static irqreturn_t hidma_chirq_handler(int chirq, void *arg) 55767a2003eSSinan Kaya { 55867a2003eSSinan Kaya struct hidma_lldev *lldev = arg; 55967a2003eSSinan Kaya 56067a2003eSSinan Kaya /* 56167a2003eSSinan Kaya * All interrupts are request driven. 56267a2003eSSinan Kaya * HW doesn't send an interrupt by itself. 56367a2003eSSinan Kaya */ 56467a2003eSSinan Kaya return hidma_ll_inthandler(chirq, lldev); 56567a2003eSSinan Kaya } 56667a2003eSSinan Kaya 5678cc12b26SArnd Bergmann #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 5681c0e3e82SSinan Kaya static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg) 5691c0e3e82SSinan Kaya { 5701c0e3e82SSinan Kaya struct hidma_lldev **lldevp = arg; 5711c0e3e82SSinan Kaya struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp); 5721c0e3e82SSinan Kaya 5731c0e3e82SSinan Kaya return hidma_ll_inthandler_msi(chirq, *lldevp, 5741c0e3e82SSinan Kaya 1 << (chirq - dmadev->msi_virqbase)); 5751c0e3e82SSinan Kaya } 5768cc12b26SArnd Bergmann #endif 5771c0e3e82SSinan Kaya 57842d236f8SSinan Kaya static ssize_t hidma_show_values(struct device *dev, 57942d236f8SSinan Kaya struct device_attribute *attr, char *buf) 58042d236f8SSinan Kaya { 58142d236f8SSinan Kaya struct platform_device *pdev = to_platform_device(dev); 58242d236f8SSinan Kaya struct hidma_dev *mdev = platform_get_drvdata(pdev); 58342d236f8SSinan Kaya 58442d236f8SSinan Kaya buf[0] = 0; 58542d236f8SSinan Kaya 58642d236f8SSinan Kaya if (strcmp(attr->attr.name, "chid") == 0) 58742d236f8SSinan Kaya sprintf(buf, "%d\n", mdev->chidx); 58842d236f8SSinan Kaya 58942d236f8SSinan Kaya return strlen(buf); 59042d236f8SSinan Kaya } 59142d236f8SSinan Kaya 592c6e4584dSSinan Kaya static inline void hidma_sysfs_uninit(struct hidma_dev *dev) 593c6e4584dSSinan Kaya { 594c6e4584dSSinan Kaya device_remove_file(dev->ddev.dev, dev->chid_attrs); 595c6e4584dSSinan Kaya } 596c6e4584dSSinan Kaya 597c6e4584dSSinan Kaya static struct device_attribute* 598c6e4584dSSinan Kaya hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode) 59942d236f8SSinan Kaya { 60042d236f8SSinan Kaya struct device_attribute *attrs; 60142d236f8SSinan Kaya char *name_copy; 60242d236f8SSinan Kaya 60342d236f8SSinan Kaya attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), 60442d236f8SSinan Kaya GFP_KERNEL); 60542d236f8SSinan Kaya if (!attrs) 606c6e4584dSSinan Kaya return NULL; 60742d236f8SSinan Kaya 60842d236f8SSinan Kaya name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); 60942d236f8SSinan Kaya if (!name_copy) 610c6e4584dSSinan Kaya return NULL; 61142d236f8SSinan Kaya 61242d236f8SSinan Kaya attrs->attr.name = name_copy; 61342d236f8SSinan Kaya attrs->attr.mode = mode; 61442d236f8SSinan Kaya attrs->show = hidma_show_values; 61542d236f8SSinan Kaya sysfs_attr_init(&attrs->attr); 61642d236f8SSinan Kaya 617c6e4584dSSinan Kaya return attrs; 618c6e4584dSSinan Kaya } 619c6e4584dSSinan Kaya 620c6e4584dSSinan Kaya static int hidma_sysfs_init(struct hidma_dev *dev) 621c6e4584dSSinan Kaya { 622c6e4584dSSinan Kaya dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO); 623c6e4584dSSinan Kaya if (!dev->chid_attrs) 624c6e4584dSSinan Kaya return -ENOMEM; 625c6e4584dSSinan Kaya 626c6e4584dSSinan Kaya return device_create_file(dev->ddev.dev, dev->chid_attrs); 62742d236f8SSinan Kaya } 62842d236f8SSinan Kaya 6291c0e3e82SSinan Kaya #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 6301c0e3e82SSinan Kaya static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 6311c0e3e82SSinan Kaya { 6321c0e3e82SSinan Kaya struct device *dev = msi_desc_to_dev(desc); 6331c0e3e82SSinan Kaya struct hidma_dev *dmadev = dev_get_drvdata(dev); 6341c0e3e82SSinan Kaya 6351c0e3e82SSinan Kaya if (!desc->platform.msi_index) { 6361c0e3e82SSinan Kaya writel(msg->address_lo, dmadev->dev_evca + 0x118); 6371c0e3e82SSinan Kaya writel(msg->address_hi, dmadev->dev_evca + 0x11C); 6381c0e3e82SSinan Kaya writel(msg->data, dmadev->dev_evca + 0x120); 6391c0e3e82SSinan Kaya } 6401c0e3e82SSinan Kaya } 6411c0e3e82SSinan Kaya #endif 6421c0e3e82SSinan Kaya 6431c0e3e82SSinan Kaya static void hidma_free_msis(struct hidma_dev *dmadev) 6441c0e3e82SSinan Kaya { 6451c0e3e82SSinan Kaya #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 6461c0e3e82SSinan Kaya struct device *dev = dmadev->ddev.dev; 6471c0e3e82SSinan Kaya struct msi_desc *desc; 6481c0e3e82SSinan Kaya 6491c0e3e82SSinan Kaya /* free allocated MSI interrupts above */ 6501c0e3e82SSinan Kaya for_each_msi_entry(desc, dev) 6511c0e3e82SSinan Kaya devm_free_irq(dev, desc->irq, &dmadev->lldev); 6521c0e3e82SSinan Kaya 6531c0e3e82SSinan Kaya platform_msi_domain_free_irqs(dev); 6541c0e3e82SSinan Kaya #endif 6551c0e3e82SSinan Kaya } 6561c0e3e82SSinan Kaya 6571c0e3e82SSinan Kaya static int hidma_request_msi(struct hidma_dev *dmadev, 6581c0e3e82SSinan Kaya struct platform_device *pdev) 6591c0e3e82SSinan Kaya { 6601c0e3e82SSinan Kaya #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 6611c0e3e82SSinan Kaya int rc; 6621c0e3e82SSinan Kaya struct msi_desc *desc; 6631c0e3e82SSinan Kaya struct msi_desc *failed_desc = NULL; 6641c0e3e82SSinan Kaya 6651c0e3e82SSinan Kaya rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, 6661c0e3e82SSinan Kaya hidma_write_msi_msg); 6671c0e3e82SSinan Kaya if (rc) 6681c0e3e82SSinan Kaya return rc; 6691c0e3e82SSinan Kaya 6701c0e3e82SSinan Kaya for_each_msi_entry(desc, &pdev->dev) { 6711c0e3e82SSinan Kaya if (!desc->platform.msi_index) 6721c0e3e82SSinan Kaya dmadev->msi_virqbase = desc->irq; 6731c0e3e82SSinan Kaya 6741c0e3e82SSinan Kaya rc = devm_request_irq(&pdev->dev, desc->irq, 6751c0e3e82SSinan Kaya hidma_chirq_handler_msi, 6761c0e3e82SSinan Kaya 0, "qcom-hidma-msi", 6771c0e3e82SSinan Kaya &dmadev->lldev); 6781c0e3e82SSinan Kaya if (rc) { 6791c0e3e82SSinan Kaya failed_desc = desc; 6801c0e3e82SSinan Kaya break; 6811c0e3e82SSinan Kaya } 6821c0e3e82SSinan Kaya } 6831c0e3e82SSinan Kaya 6841c0e3e82SSinan Kaya if (rc) { 6851c0e3e82SSinan Kaya /* free allocated MSI interrupts above */ 6861c0e3e82SSinan Kaya for_each_msi_entry(desc, &pdev->dev) { 6871c0e3e82SSinan Kaya if (desc == failed_desc) 6881c0e3e82SSinan Kaya break; 6891c0e3e82SSinan Kaya devm_free_irq(&pdev->dev, desc->irq, 6901c0e3e82SSinan Kaya &dmadev->lldev); 6911c0e3e82SSinan Kaya } 6921c0e3e82SSinan Kaya } else { 6931c0e3e82SSinan Kaya /* Add callback to free MSIs on teardown */ 6941c0e3e82SSinan Kaya hidma_ll_setup_irq(dmadev->lldev, true); 6951c0e3e82SSinan Kaya 6961c0e3e82SSinan Kaya } 6971c0e3e82SSinan Kaya if (rc) 6981c0e3e82SSinan Kaya dev_warn(&pdev->dev, 6991c0e3e82SSinan Kaya "failed to request MSI irq, falling back to wired IRQ\n"); 7001c0e3e82SSinan Kaya return rc; 7011c0e3e82SSinan Kaya #else 7021c0e3e82SSinan Kaya return -EINVAL; 7031c0e3e82SSinan Kaya #endif 7041c0e3e82SSinan Kaya } 7051c0e3e82SSinan Kaya 7061c0e3e82SSinan Kaya static bool hidma_msi_capable(struct device *dev) 7071c0e3e82SSinan Kaya { 7081c0e3e82SSinan Kaya struct acpi_device *adev = ACPI_COMPANION(dev); 7091c0e3e82SSinan Kaya const char *of_compat; 7101c0e3e82SSinan Kaya int ret = -EINVAL; 7111c0e3e82SSinan Kaya 7121c0e3e82SSinan Kaya if (!adev || acpi_disabled) { 7131c0e3e82SSinan Kaya ret = device_property_read_string(dev, "compatible", 7141c0e3e82SSinan Kaya &of_compat); 7151c0e3e82SSinan Kaya if (ret) 7161c0e3e82SSinan Kaya return false; 7171c0e3e82SSinan Kaya 7181c0e3e82SSinan Kaya ret = strcmp(of_compat, "qcom,hidma-1.1"); 7191c0e3e82SSinan Kaya } else { 7201c0e3e82SSinan Kaya #ifdef CONFIG_ACPI 7211c0e3e82SSinan Kaya ret = strcmp(acpi_device_hid(adev), "QCOM8062"); 7221c0e3e82SSinan Kaya #endif 7231c0e3e82SSinan Kaya } 7241c0e3e82SSinan Kaya return ret == 0; 7251c0e3e82SSinan Kaya } 7261c0e3e82SSinan Kaya 72767a2003eSSinan Kaya static int hidma_probe(struct platform_device *pdev) 72867a2003eSSinan Kaya { 72967a2003eSSinan Kaya struct hidma_dev *dmadev; 73067a2003eSSinan Kaya struct resource *trca_resource; 73167a2003eSSinan Kaya struct resource *evca_resource; 73267a2003eSSinan Kaya int chirq; 73367a2003eSSinan Kaya void __iomem *evca; 73467a2003eSSinan Kaya void __iomem *trca; 73567a2003eSSinan Kaya int rc; 7361c0e3e82SSinan Kaya bool msi; 73767a2003eSSinan Kaya 73867a2003eSSinan Kaya pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); 73967a2003eSSinan Kaya pm_runtime_use_autosuspend(&pdev->dev); 74067a2003eSSinan Kaya pm_runtime_set_active(&pdev->dev); 74167a2003eSSinan Kaya pm_runtime_enable(&pdev->dev); 74267a2003eSSinan Kaya 74367a2003eSSinan Kaya trca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); 74467a2003eSSinan Kaya trca = devm_ioremap_resource(&pdev->dev, trca_resource); 74567a2003eSSinan Kaya if (IS_ERR(trca)) { 74667a2003eSSinan Kaya rc = -ENOMEM; 74767a2003eSSinan Kaya goto bailout; 74867a2003eSSinan Kaya } 74967a2003eSSinan Kaya 75067a2003eSSinan Kaya evca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 1); 75167a2003eSSinan Kaya evca = devm_ioremap_resource(&pdev->dev, evca_resource); 75267a2003eSSinan Kaya if (IS_ERR(evca)) { 75367a2003eSSinan Kaya rc = -ENOMEM; 75467a2003eSSinan Kaya goto bailout; 75567a2003eSSinan Kaya } 75667a2003eSSinan Kaya 75767a2003eSSinan Kaya /* 75867a2003eSSinan Kaya * This driver only handles the channel IRQs. 75967a2003eSSinan Kaya * Common IRQ is handled by the management driver. 76067a2003eSSinan Kaya */ 76167a2003eSSinan Kaya chirq = platform_get_irq(pdev, 0); 76267a2003eSSinan Kaya if (chirq < 0) { 76367a2003eSSinan Kaya rc = -ENODEV; 76467a2003eSSinan Kaya goto bailout; 76567a2003eSSinan Kaya } 76667a2003eSSinan Kaya 76767a2003eSSinan Kaya dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); 76867a2003eSSinan Kaya if (!dmadev) { 76967a2003eSSinan Kaya rc = -ENOMEM; 77067a2003eSSinan Kaya goto bailout; 77167a2003eSSinan Kaya } 77267a2003eSSinan Kaya 77367a2003eSSinan Kaya INIT_LIST_HEAD(&dmadev->ddev.channels); 77467a2003eSSinan Kaya spin_lock_init(&dmadev->lock); 77567a2003eSSinan Kaya dmadev->ddev.dev = &pdev->dev; 77667a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 77767a2003eSSinan Kaya 77867a2003eSSinan Kaya dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); 77967a2003eSSinan Kaya if (WARN_ON(!pdev->dev.dma_mask)) { 78067a2003eSSinan Kaya rc = -ENXIO; 78167a2003eSSinan Kaya goto dmafree; 78267a2003eSSinan Kaya } 78367a2003eSSinan Kaya 78467a2003eSSinan Kaya dmadev->dev_evca = evca; 78567a2003eSSinan Kaya dmadev->evca_resource = evca_resource; 78667a2003eSSinan Kaya dmadev->dev_trca = trca; 78767a2003eSSinan Kaya dmadev->trca_resource = trca_resource; 78867a2003eSSinan Kaya dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; 78967a2003eSSinan Kaya dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; 79067a2003eSSinan Kaya dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; 79167a2003eSSinan Kaya dmadev->ddev.device_tx_status = hidma_tx_status; 79267a2003eSSinan Kaya dmadev->ddev.device_issue_pending = hidma_issue_pending; 79367a2003eSSinan Kaya dmadev->ddev.device_pause = hidma_pause; 79467a2003eSSinan Kaya dmadev->ddev.device_resume = hidma_resume; 79567a2003eSSinan Kaya dmadev->ddev.device_terminate_all = hidma_terminate_all; 79667a2003eSSinan Kaya dmadev->ddev.copy_align = 8; 79767a2003eSSinan Kaya 7981c0e3e82SSinan Kaya /* 7991c0e3e82SSinan Kaya * Determine the MSI capability of the platform. Old HW doesn't 8001c0e3e82SSinan Kaya * support MSI. 8011c0e3e82SSinan Kaya */ 8021c0e3e82SSinan Kaya msi = hidma_msi_capable(&pdev->dev); 8031c0e3e82SSinan Kaya 80467a2003eSSinan Kaya device_property_read_u32(&pdev->dev, "desc-count", 80567a2003eSSinan Kaya &dmadev->nr_descriptors); 80667a2003eSSinan Kaya 80713058e33SSinan Kaya if (nr_desc_prm) { 80813058e33SSinan Kaya dev_info(&pdev->dev, "overriding number of descriptors as %d\n", 80913058e33SSinan Kaya nr_desc_prm); 81067a2003eSSinan Kaya dmadev->nr_descriptors = nr_desc_prm; 81113058e33SSinan Kaya } 81267a2003eSSinan Kaya 81367a2003eSSinan Kaya if (!dmadev->nr_descriptors) 81467a2003eSSinan Kaya dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; 81567a2003eSSinan Kaya 81667a2003eSSinan Kaya dmadev->chidx = readl(dmadev->dev_trca + 0x28); 81767a2003eSSinan Kaya 81867a2003eSSinan Kaya /* Set DMA mask to 64 bits. */ 81967a2003eSSinan Kaya rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 82067a2003eSSinan Kaya if (rc) { 82167a2003eSSinan Kaya dev_warn(&pdev->dev, "unable to set coherent mask to 64"); 82267a2003eSSinan Kaya rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 82367a2003eSSinan Kaya if (rc) 82467a2003eSSinan Kaya goto dmafree; 82567a2003eSSinan Kaya } 82667a2003eSSinan Kaya 82767a2003eSSinan Kaya dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, 82867a2003eSSinan Kaya dmadev->nr_descriptors, dmadev->dev_trca, 82967a2003eSSinan Kaya dmadev->dev_evca, dmadev->chidx); 83067a2003eSSinan Kaya if (!dmadev->lldev) { 83167a2003eSSinan Kaya rc = -EPROBE_DEFER; 83267a2003eSSinan Kaya goto dmafree; 83367a2003eSSinan Kaya } 83467a2003eSSinan Kaya 8351c0e3e82SSinan Kaya platform_set_drvdata(pdev, dmadev); 8361c0e3e82SSinan Kaya if (msi) 8371c0e3e82SSinan Kaya rc = hidma_request_msi(dmadev, pdev); 8381c0e3e82SSinan Kaya 8391c0e3e82SSinan Kaya if (!msi || rc) { 8401c0e3e82SSinan Kaya hidma_ll_setup_irq(dmadev->lldev, false); 8411c0e3e82SSinan Kaya rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, 8421c0e3e82SSinan Kaya 0, "qcom-hidma", dmadev->lldev); 84367a2003eSSinan Kaya if (rc) 84467a2003eSSinan Kaya goto uninit; 8451c0e3e82SSinan Kaya } 84667a2003eSSinan Kaya 84767a2003eSSinan Kaya INIT_LIST_HEAD(&dmadev->ddev.channels); 84867a2003eSSinan Kaya rc = hidma_chan_init(dmadev, 0); 84967a2003eSSinan Kaya if (rc) 85067a2003eSSinan Kaya goto uninit; 85167a2003eSSinan Kaya 85267a2003eSSinan Kaya rc = dma_async_device_register(&dmadev->ddev); 85367a2003eSSinan Kaya if (rc) 85467a2003eSSinan Kaya goto uninit; 85567a2003eSSinan Kaya 85667a2003eSSinan Kaya dmadev->irq = chirq; 85767a2003eSSinan Kaya tasklet_init(&dmadev->task, hidma_issue_task, (unsigned long)dmadev); 858570d0176SSinan Kaya hidma_debug_init(dmadev); 859c6e4584dSSinan Kaya hidma_sysfs_init(dmadev); 86067a2003eSSinan Kaya dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); 86167a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 86267a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 86367a2003eSSinan Kaya return 0; 86467a2003eSSinan Kaya 86567a2003eSSinan Kaya uninit: 8661c0e3e82SSinan Kaya if (msi) 8671c0e3e82SSinan Kaya hidma_free_msis(dmadev); 8681c0e3e82SSinan Kaya 869570d0176SSinan Kaya hidma_debug_uninit(dmadev); 87067a2003eSSinan Kaya hidma_ll_uninit(dmadev->lldev); 87167a2003eSSinan Kaya dmafree: 87267a2003eSSinan Kaya if (dmadev) 87367a2003eSSinan Kaya hidma_free(dmadev); 87467a2003eSSinan Kaya bailout: 87567a2003eSSinan Kaya pm_runtime_put_sync(&pdev->dev); 87667a2003eSSinan Kaya pm_runtime_disable(&pdev->dev); 87767a2003eSSinan Kaya return rc; 87867a2003eSSinan Kaya } 87967a2003eSSinan Kaya 880dc7c733aSSinan Kaya static void hidma_shutdown(struct platform_device *pdev) 881dc7c733aSSinan Kaya { 882dc7c733aSSinan Kaya struct hidma_dev *dmadev = platform_get_drvdata(pdev); 883dc7c733aSSinan Kaya 884dc7c733aSSinan Kaya dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n"); 885dc7c733aSSinan Kaya 886dc7c733aSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 887dc7c733aSSinan Kaya if (hidma_ll_disable(dmadev->lldev)) 888dc7c733aSSinan Kaya dev_warn(dmadev->ddev.dev, "channel did not stop\n"); 889dc7c733aSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 890dc7c733aSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 891dc7c733aSSinan Kaya 892dc7c733aSSinan Kaya } 893dc7c733aSSinan Kaya 89467a2003eSSinan Kaya static int hidma_remove(struct platform_device *pdev) 89567a2003eSSinan Kaya { 89667a2003eSSinan Kaya struct hidma_dev *dmadev = platform_get_drvdata(pdev); 89767a2003eSSinan Kaya 89867a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 89967a2003eSSinan Kaya dma_async_device_unregister(&dmadev->ddev); 9001c0e3e82SSinan Kaya if (!dmadev->lldev->msi_support) 90167a2003eSSinan Kaya devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); 9021c0e3e82SSinan Kaya else 9031c0e3e82SSinan Kaya hidma_free_msis(dmadev); 9041c0e3e82SSinan Kaya 905bd16934aSVinod Koul tasklet_kill(&dmadev->task); 906c6e4584dSSinan Kaya hidma_sysfs_uninit(dmadev); 907570d0176SSinan Kaya hidma_debug_uninit(dmadev); 90867a2003eSSinan Kaya hidma_ll_uninit(dmadev->lldev); 90967a2003eSSinan Kaya hidma_free(dmadev); 91067a2003eSSinan Kaya 91167a2003eSSinan Kaya dev_info(&pdev->dev, "HI-DMA engine removed\n"); 91267a2003eSSinan Kaya pm_runtime_put_sync_suspend(&pdev->dev); 91367a2003eSSinan Kaya pm_runtime_disable(&pdev->dev); 91467a2003eSSinan Kaya 91567a2003eSSinan Kaya return 0; 91667a2003eSSinan Kaya } 91767a2003eSSinan Kaya 91867a2003eSSinan Kaya #if IS_ENABLED(CONFIG_ACPI) 91967a2003eSSinan Kaya static const struct acpi_device_id hidma_acpi_ids[] = { 92067a2003eSSinan Kaya {"QCOM8061"}, 9211c0e3e82SSinan Kaya {"QCOM8062"}, 92267a2003eSSinan Kaya {}, 92367a2003eSSinan Kaya }; 92475ff7668SSinan Kaya MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids); 92567a2003eSSinan Kaya #endif 92667a2003eSSinan Kaya 92767a2003eSSinan Kaya static const struct of_device_id hidma_match[] = { 92867a2003eSSinan Kaya {.compatible = "qcom,hidma-1.0",}, 9291c0e3e82SSinan Kaya {.compatible = "qcom,hidma-1.1",}, 93067a2003eSSinan Kaya {}, 93167a2003eSSinan Kaya }; 93267a2003eSSinan Kaya MODULE_DEVICE_TABLE(of, hidma_match); 93367a2003eSSinan Kaya 93467a2003eSSinan Kaya static struct platform_driver hidma_driver = { 93567a2003eSSinan Kaya .probe = hidma_probe, 93667a2003eSSinan Kaya .remove = hidma_remove, 937dc7c733aSSinan Kaya .shutdown = hidma_shutdown, 93867a2003eSSinan Kaya .driver = { 93967a2003eSSinan Kaya .name = "hidma", 94067a2003eSSinan Kaya .of_match_table = hidma_match, 94167a2003eSSinan Kaya .acpi_match_table = ACPI_PTR(hidma_acpi_ids), 94267a2003eSSinan Kaya }, 94367a2003eSSinan Kaya }; 94467a2003eSSinan Kaya 94567a2003eSSinan Kaya module_platform_driver(hidma_driver); 94667a2003eSSinan Kaya MODULE_LICENSE("GPL v2"); 947