167a2003eSSinan Kaya /* 267a2003eSSinan Kaya * Qualcomm Technologies HIDMA DMA engine interface 367a2003eSSinan Kaya * 4570d0176SSinan Kaya * Copyright (c) 2015-2016, The Linux Foundation. All rights reserved. 567a2003eSSinan Kaya * 667a2003eSSinan Kaya * This program is free software; you can redistribute it and/or modify 767a2003eSSinan Kaya * it under the terms of the GNU General Public License version 2 and 867a2003eSSinan Kaya * only version 2 as published by the Free Software Foundation. 967a2003eSSinan Kaya * 1067a2003eSSinan Kaya * This program is distributed in the hope that it will be useful, 1167a2003eSSinan Kaya * but WITHOUT ANY WARRANTY; without even the implied warranty of 1267a2003eSSinan Kaya * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1367a2003eSSinan Kaya * GNU General Public License for more details. 1467a2003eSSinan Kaya */ 1567a2003eSSinan Kaya 1667a2003eSSinan Kaya /* 1767a2003eSSinan Kaya * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008. 1867a2003eSSinan Kaya * Copyright (C) Semihalf 2009 1967a2003eSSinan Kaya * Copyright (C) Ilya Yanok, Emcraft Systems 2010 2067a2003eSSinan Kaya * Copyright (C) Alexander Popov, Promcontroller 2014 2167a2003eSSinan Kaya * 2267a2003eSSinan Kaya * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description 2367a2003eSSinan Kaya * (defines, structures and comments) was taken from MPC5121 DMA driver 2467a2003eSSinan Kaya * written by Hongjun Chen <hong-jun.chen@freescale.com>. 2567a2003eSSinan Kaya * 2667a2003eSSinan Kaya * Approved as OSADL project by a majority of OSADL members and funded 2767a2003eSSinan Kaya * by OSADL membership fees in 2009; for details see www.osadl.org. 2867a2003eSSinan Kaya * 2967a2003eSSinan Kaya * This program is free software; you can redistribute it and/or modify it 3067a2003eSSinan Kaya * under the terms of the GNU General Public License as published by the Free 3167a2003eSSinan Kaya * Software Foundation; either version 2 of the License, or (at your option) 3267a2003eSSinan Kaya * any later version. 3367a2003eSSinan Kaya * 3467a2003eSSinan Kaya * This program is distributed in the hope that it will be useful, but WITHOUT 3567a2003eSSinan Kaya * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 3667a2003eSSinan Kaya * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 3767a2003eSSinan Kaya * more details. 3867a2003eSSinan Kaya * 3967a2003eSSinan Kaya * The full GNU General Public License is included in this distribution in the 4067a2003eSSinan Kaya * file called COPYING. 4167a2003eSSinan Kaya */ 4267a2003eSSinan Kaya 4367a2003eSSinan Kaya /* Linux Foundation elects GPLv2 license only. */ 4467a2003eSSinan Kaya 4567a2003eSSinan Kaya #include <linux/dmaengine.h> 4667a2003eSSinan Kaya #include <linux/dma-mapping.h> 4767a2003eSSinan Kaya #include <linux/list.h> 4867a2003eSSinan Kaya #include <linux/module.h> 4967a2003eSSinan Kaya #include <linux/platform_device.h> 5067a2003eSSinan Kaya #include <linux/slab.h> 5167a2003eSSinan Kaya #include <linux/spinlock.h> 5267a2003eSSinan Kaya #include <linux/of_dma.h> 5367a2003eSSinan Kaya #include <linux/property.h> 5467a2003eSSinan Kaya #include <linux/delay.h> 5567a2003eSSinan Kaya #include <linux/acpi.h> 5667a2003eSSinan Kaya #include <linux/irq.h> 5767a2003eSSinan Kaya #include <linux/atomic.h> 5867a2003eSSinan Kaya #include <linux/pm_runtime.h> 59*1c0e3e82SSinan Kaya #include <linux/msi.h> 6067a2003eSSinan Kaya 6167a2003eSSinan Kaya #include "../dmaengine.h" 6267a2003eSSinan Kaya #include "hidma.h" 6367a2003eSSinan Kaya 6467a2003eSSinan Kaya /* 6567a2003eSSinan Kaya * Default idle time is 2 seconds. This parameter can 6667a2003eSSinan Kaya * be overridden by changing the following 6767a2003eSSinan Kaya * /sys/bus/platform/devices/QCOM8061:<xy>/power/autosuspend_delay_ms 6867a2003eSSinan Kaya * during kernel boot. 6967a2003eSSinan Kaya */ 7067a2003eSSinan Kaya #define HIDMA_AUTOSUSPEND_TIMEOUT 2000 7167a2003eSSinan Kaya #define HIDMA_ERR_INFO_SW 0xFF 7267a2003eSSinan Kaya #define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0 7367a2003eSSinan Kaya #define HIDMA_NR_DEFAULT_DESC 10 74*1c0e3e82SSinan Kaya #define HIDMA_MSI_INTS 11 7567a2003eSSinan Kaya 7667a2003eSSinan Kaya static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev) 7767a2003eSSinan Kaya { 7867a2003eSSinan Kaya return container_of(dmadev, struct hidma_dev, ddev); 7967a2003eSSinan Kaya } 8067a2003eSSinan Kaya 8167a2003eSSinan Kaya static inline 8267a2003eSSinan Kaya struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp) 8367a2003eSSinan Kaya { 8467a2003eSSinan Kaya return container_of(_lldevp, struct hidma_dev, lldev); 8567a2003eSSinan Kaya } 8667a2003eSSinan Kaya 8767a2003eSSinan Kaya static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach) 8867a2003eSSinan Kaya { 8967a2003eSSinan Kaya return container_of(dmach, struct hidma_chan, chan); 9067a2003eSSinan Kaya } 9167a2003eSSinan Kaya 9267a2003eSSinan Kaya static inline 9367a2003eSSinan Kaya struct hidma_desc *to_hidma_desc(struct dma_async_tx_descriptor *t) 9467a2003eSSinan Kaya { 9567a2003eSSinan Kaya return container_of(t, struct hidma_desc, desc); 9667a2003eSSinan Kaya } 9767a2003eSSinan Kaya 9867a2003eSSinan Kaya static void hidma_free(struct hidma_dev *dmadev) 9967a2003eSSinan Kaya { 10067a2003eSSinan Kaya INIT_LIST_HEAD(&dmadev->ddev.channels); 10167a2003eSSinan Kaya } 10267a2003eSSinan Kaya 10367a2003eSSinan Kaya static unsigned int nr_desc_prm; 10467a2003eSSinan Kaya module_param(nr_desc_prm, uint, 0644); 10567a2003eSSinan Kaya MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)"); 10667a2003eSSinan Kaya 10767a2003eSSinan Kaya 10867a2003eSSinan Kaya /* process completed descriptors */ 10967a2003eSSinan Kaya static void hidma_process_completed(struct hidma_chan *mchan) 11067a2003eSSinan Kaya { 11167a2003eSSinan Kaya struct dma_device *ddev = mchan->chan.device; 11267a2003eSSinan Kaya struct hidma_dev *mdma = to_hidma_dev(ddev); 11367a2003eSSinan Kaya struct dma_async_tx_descriptor *desc; 11467a2003eSSinan Kaya dma_cookie_t last_cookie; 11567a2003eSSinan Kaya struct hidma_desc *mdesc; 1168a31f8b5SSinan Kaya struct hidma_desc *next; 11767a2003eSSinan Kaya unsigned long irqflags; 11867a2003eSSinan Kaya struct list_head list; 11967a2003eSSinan Kaya 12067a2003eSSinan Kaya INIT_LIST_HEAD(&list); 12167a2003eSSinan Kaya 12267a2003eSSinan Kaya /* Get all completed descriptors */ 12367a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 12467a2003eSSinan Kaya list_splice_tail_init(&mchan->completed, &list); 12567a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 12667a2003eSSinan Kaya 12767a2003eSSinan Kaya /* Execute callbacks and run dependencies */ 1288a31f8b5SSinan Kaya list_for_each_entry_safe(mdesc, next, &list, node) { 12967a2003eSSinan Kaya enum dma_status llstat; 1308a31f8b5SSinan Kaya struct dmaengine_desc_callback cb; 13155c370e5SSinan Kaya struct dmaengine_result result; 13267a2003eSSinan Kaya 13367a2003eSSinan Kaya desc = &mdesc->desc; 134793ae66cSSinan Kaya last_cookie = desc->cookie; 13567a2003eSSinan Kaya 13667a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 13767a2003eSSinan Kaya dma_cookie_complete(desc); 13867a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 13967a2003eSSinan Kaya 14067a2003eSSinan Kaya llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); 1418a31f8b5SSinan Kaya dmaengine_desc_get_callback(desc, &cb); 14267a2003eSSinan Kaya 14367a2003eSSinan Kaya dma_run_dependencies(desc); 14467a2003eSSinan Kaya 14567a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 1468a31f8b5SSinan Kaya list_move(&mdesc->node, &mchan->free); 14767a2003eSSinan Kaya 148793ae66cSSinan Kaya if (llstat == DMA_COMPLETE) { 149793ae66cSSinan Kaya mchan->last_success = last_cookie; 15055c370e5SSinan Kaya result.result = DMA_TRANS_NOERROR; 151793ae66cSSinan Kaya } else 15255c370e5SSinan Kaya result.result = DMA_TRANS_ABORTED; 15355c370e5SSinan Kaya 15455c370e5SSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 15555c370e5SSinan Kaya 15655c370e5SSinan Kaya dmaengine_desc_callback_invoke(&cb, &result); 1578a31f8b5SSinan Kaya } 15867a2003eSSinan Kaya } 15967a2003eSSinan Kaya 16067a2003eSSinan Kaya /* 16167a2003eSSinan Kaya * Called once for each submitted descriptor. 16267a2003eSSinan Kaya * PM is locked once for each descriptor that is currently 16367a2003eSSinan Kaya * in execution. 16467a2003eSSinan Kaya */ 16567a2003eSSinan Kaya static void hidma_callback(void *data) 16667a2003eSSinan Kaya { 16767a2003eSSinan Kaya struct hidma_desc *mdesc = data; 16867a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); 16967a2003eSSinan Kaya struct dma_device *ddev = mchan->chan.device; 17067a2003eSSinan Kaya struct hidma_dev *dmadev = to_hidma_dev(ddev); 17167a2003eSSinan Kaya unsigned long irqflags; 17267a2003eSSinan Kaya bool queued = false; 17367a2003eSSinan Kaya 17467a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 17567a2003eSSinan Kaya if (mdesc->node.next) { 17667a2003eSSinan Kaya /* Delete from the active list, add to completed list */ 17767a2003eSSinan Kaya list_move_tail(&mdesc->node, &mchan->completed); 17867a2003eSSinan Kaya queued = true; 17967a2003eSSinan Kaya 18067a2003eSSinan Kaya /* calculate the next running descriptor */ 18167a2003eSSinan Kaya mchan->running = list_first_entry(&mchan->active, 18267a2003eSSinan Kaya struct hidma_desc, node); 18367a2003eSSinan Kaya } 18467a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 18567a2003eSSinan Kaya 18667a2003eSSinan Kaya hidma_process_completed(mchan); 18767a2003eSSinan Kaya 18867a2003eSSinan Kaya if (queued) { 18967a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 19067a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 19167a2003eSSinan Kaya } 19267a2003eSSinan Kaya } 19367a2003eSSinan Kaya 19467a2003eSSinan Kaya static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig) 19567a2003eSSinan Kaya { 19667a2003eSSinan Kaya struct hidma_chan *mchan; 19767a2003eSSinan Kaya struct dma_device *ddev; 19867a2003eSSinan Kaya 19967a2003eSSinan Kaya mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); 20067a2003eSSinan Kaya if (!mchan) 20167a2003eSSinan Kaya return -ENOMEM; 20267a2003eSSinan Kaya 20367a2003eSSinan Kaya ddev = &dmadev->ddev; 20467a2003eSSinan Kaya mchan->dma_sig = dma_sig; 20567a2003eSSinan Kaya mchan->dmadev = dmadev; 20667a2003eSSinan Kaya mchan->chan.device = ddev; 20767a2003eSSinan Kaya dma_cookie_init(&mchan->chan); 20867a2003eSSinan Kaya 20967a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->free); 21067a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->prepared); 21167a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->active); 21267a2003eSSinan Kaya INIT_LIST_HEAD(&mchan->completed); 21367a2003eSSinan Kaya 21467a2003eSSinan Kaya spin_lock_init(&mchan->lock); 21567a2003eSSinan Kaya list_add_tail(&mchan->chan.device_node, &ddev->channels); 21667a2003eSSinan Kaya dmadev->ddev.chancnt++; 21767a2003eSSinan Kaya return 0; 21867a2003eSSinan Kaya } 21967a2003eSSinan Kaya 22067a2003eSSinan Kaya static void hidma_issue_task(unsigned long arg) 22167a2003eSSinan Kaya { 22267a2003eSSinan Kaya struct hidma_dev *dmadev = (struct hidma_dev *)arg; 22367a2003eSSinan Kaya 22467a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 22567a2003eSSinan Kaya hidma_ll_start(dmadev->lldev); 22667a2003eSSinan Kaya } 22767a2003eSSinan Kaya 22867a2003eSSinan Kaya static void hidma_issue_pending(struct dma_chan *dmach) 22967a2003eSSinan Kaya { 23067a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 23167a2003eSSinan Kaya struct hidma_dev *dmadev = mchan->dmadev; 23267a2003eSSinan Kaya unsigned long flags; 23367a2003eSSinan Kaya int status; 23467a2003eSSinan Kaya 23567a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, flags); 23667a2003eSSinan Kaya if (!mchan->running) { 23767a2003eSSinan Kaya struct hidma_desc *desc = list_first_entry(&mchan->active, 23867a2003eSSinan Kaya struct hidma_desc, 23967a2003eSSinan Kaya node); 24067a2003eSSinan Kaya mchan->running = desc; 24167a2003eSSinan Kaya } 24267a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, flags); 24367a2003eSSinan Kaya 24467a2003eSSinan Kaya /* PM will be released in hidma_callback function. */ 24567a2003eSSinan Kaya status = pm_runtime_get(dmadev->ddev.dev); 24667a2003eSSinan Kaya if (status < 0) 24767a2003eSSinan Kaya tasklet_schedule(&dmadev->task); 24867a2003eSSinan Kaya else 24967a2003eSSinan Kaya hidma_ll_start(dmadev->lldev); 25067a2003eSSinan Kaya } 25167a2003eSSinan Kaya 252793ae66cSSinan Kaya static inline bool hidma_txn_is_success(dma_cookie_t cookie, 253793ae66cSSinan Kaya dma_cookie_t last_success, dma_cookie_t last_used) 254793ae66cSSinan Kaya { 255793ae66cSSinan Kaya if (last_success <= last_used) { 256793ae66cSSinan Kaya if ((cookie <= last_success) || (cookie > last_used)) 257793ae66cSSinan Kaya return true; 258793ae66cSSinan Kaya } else { 259793ae66cSSinan Kaya if ((cookie <= last_success) && (cookie > last_used)) 260793ae66cSSinan Kaya return true; 261793ae66cSSinan Kaya } 262793ae66cSSinan Kaya return false; 263793ae66cSSinan Kaya } 264793ae66cSSinan Kaya 26567a2003eSSinan Kaya static enum dma_status hidma_tx_status(struct dma_chan *dmach, 26667a2003eSSinan Kaya dma_cookie_t cookie, 26767a2003eSSinan Kaya struct dma_tx_state *txstate) 26867a2003eSSinan Kaya { 26967a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 27067a2003eSSinan Kaya enum dma_status ret; 27167a2003eSSinan Kaya 27267a2003eSSinan Kaya ret = dma_cookie_status(dmach, cookie, txstate); 273793ae66cSSinan Kaya if (ret == DMA_COMPLETE) { 274793ae66cSSinan Kaya bool is_success; 275793ae66cSSinan Kaya 276793ae66cSSinan Kaya is_success = hidma_txn_is_success(cookie, mchan->last_success, 277793ae66cSSinan Kaya dmach->cookie); 278793ae66cSSinan Kaya return is_success ? ret : DMA_ERROR; 279793ae66cSSinan Kaya } 28067a2003eSSinan Kaya 28167a2003eSSinan Kaya if (mchan->paused && (ret == DMA_IN_PROGRESS)) { 28267a2003eSSinan Kaya unsigned long flags; 28367a2003eSSinan Kaya dma_cookie_t runcookie; 28467a2003eSSinan Kaya 28567a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, flags); 28667a2003eSSinan Kaya if (mchan->running) 28767a2003eSSinan Kaya runcookie = mchan->running->desc.cookie; 28867a2003eSSinan Kaya else 28967a2003eSSinan Kaya runcookie = -EINVAL; 29067a2003eSSinan Kaya 29167a2003eSSinan Kaya if (runcookie == cookie) 29267a2003eSSinan Kaya ret = DMA_PAUSED; 29367a2003eSSinan Kaya 29467a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, flags); 29567a2003eSSinan Kaya } 29667a2003eSSinan Kaya 29767a2003eSSinan Kaya return ret; 29867a2003eSSinan Kaya } 29967a2003eSSinan Kaya 30067a2003eSSinan Kaya /* 30167a2003eSSinan Kaya * Submit descriptor to hardware. 30267a2003eSSinan Kaya * Lock the PM for each descriptor we are sending. 30367a2003eSSinan Kaya */ 30467a2003eSSinan Kaya static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd) 30567a2003eSSinan Kaya { 30667a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(txd->chan); 30767a2003eSSinan Kaya struct hidma_dev *dmadev = mchan->dmadev; 30867a2003eSSinan Kaya struct hidma_desc *mdesc; 30967a2003eSSinan Kaya unsigned long irqflags; 31067a2003eSSinan Kaya dma_cookie_t cookie; 31167a2003eSSinan Kaya 31267a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 31367a2003eSSinan Kaya if (!hidma_ll_isenabled(dmadev->lldev)) { 31467a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 31567a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 31667a2003eSSinan Kaya return -ENODEV; 31767a2003eSSinan Kaya } 31867a2003eSSinan Kaya 31967a2003eSSinan Kaya mdesc = container_of(txd, struct hidma_desc, desc); 32067a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 32167a2003eSSinan Kaya 32267a2003eSSinan Kaya /* Move descriptor to active */ 32367a2003eSSinan Kaya list_move_tail(&mdesc->node, &mchan->active); 32467a2003eSSinan Kaya 32567a2003eSSinan Kaya /* Update cookie */ 32667a2003eSSinan Kaya cookie = dma_cookie_assign(txd); 32767a2003eSSinan Kaya 32867a2003eSSinan Kaya hidma_ll_queue_request(dmadev->lldev, mdesc->tre_ch); 32967a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 33067a2003eSSinan Kaya 33167a2003eSSinan Kaya return cookie; 33267a2003eSSinan Kaya } 33367a2003eSSinan Kaya 33467a2003eSSinan Kaya static int hidma_alloc_chan_resources(struct dma_chan *dmach) 33567a2003eSSinan Kaya { 33667a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 33767a2003eSSinan Kaya struct hidma_dev *dmadev = mchan->dmadev; 33867a2003eSSinan Kaya struct hidma_desc *mdesc, *tmp; 33967a2003eSSinan Kaya unsigned long irqflags; 34067a2003eSSinan Kaya LIST_HEAD(descs); 34167a2003eSSinan Kaya unsigned int i; 34267a2003eSSinan Kaya int rc = 0; 34367a2003eSSinan Kaya 34467a2003eSSinan Kaya if (mchan->allocated) 34567a2003eSSinan Kaya return 0; 34667a2003eSSinan Kaya 34767a2003eSSinan Kaya /* Alloc descriptors for this channel */ 34867a2003eSSinan Kaya for (i = 0; i < dmadev->nr_descriptors; i++) { 34967a2003eSSinan Kaya mdesc = kzalloc(sizeof(struct hidma_desc), GFP_NOWAIT); 35067a2003eSSinan Kaya if (!mdesc) { 35167a2003eSSinan Kaya rc = -ENOMEM; 35267a2003eSSinan Kaya break; 35367a2003eSSinan Kaya } 35467a2003eSSinan Kaya dma_async_tx_descriptor_init(&mdesc->desc, dmach); 35567a2003eSSinan Kaya mdesc->desc.tx_submit = hidma_tx_submit; 35667a2003eSSinan Kaya 35767a2003eSSinan Kaya rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, 35867a2003eSSinan Kaya "DMA engine", hidma_callback, mdesc, 35967a2003eSSinan Kaya &mdesc->tre_ch); 36067a2003eSSinan Kaya if (rc) { 36167a2003eSSinan Kaya dev_err(dmach->device->dev, 36267a2003eSSinan Kaya "channel alloc failed at %u\n", i); 36367a2003eSSinan Kaya kfree(mdesc); 36467a2003eSSinan Kaya break; 36567a2003eSSinan Kaya } 36667a2003eSSinan Kaya list_add_tail(&mdesc->node, &descs); 36767a2003eSSinan Kaya } 36867a2003eSSinan Kaya 36967a2003eSSinan Kaya if (rc) { 37067a2003eSSinan Kaya /* return the allocated descriptors */ 37167a2003eSSinan Kaya list_for_each_entry_safe(mdesc, tmp, &descs, node) { 37267a2003eSSinan Kaya hidma_ll_free(dmadev->lldev, mdesc->tre_ch); 37367a2003eSSinan Kaya kfree(mdesc); 37467a2003eSSinan Kaya } 37567a2003eSSinan Kaya return rc; 37667a2003eSSinan Kaya } 37767a2003eSSinan Kaya 37867a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 37967a2003eSSinan Kaya list_splice_tail_init(&descs, &mchan->free); 38067a2003eSSinan Kaya mchan->allocated = true; 38167a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 38267a2003eSSinan Kaya return 1; 38367a2003eSSinan Kaya } 38467a2003eSSinan Kaya 38567a2003eSSinan Kaya static struct dma_async_tx_descriptor * 38667a2003eSSinan Kaya hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src, 38767a2003eSSinan Kaya size_t len, unsigned long flags) 38867a2003eSSinan Kaya { 38967a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 39067a2003eSSinan Kaya struct hidma_desc *mdesc = NULL; 39167a2003eSSinan Kaya struct hidma_dev *mdma = mchan->dmadev; 39267a2003eSSinan Kaya unsigned long irqflags; 39367a2003eSSinan Kaya 39467a2003eSSinan Kaya /* Get free descriptor */ 39567a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 39667a2003eSSinan Kaya if (!list_empty(&mchan->free)) { 39767a2003eSSinan Kaya mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); 39867a2003eSSinan Kaya list_del(&mdesc->node); 39967a2003eSSinan Kaya } 40067a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 40167a2003eSSinan Kaya 40267a2003eSSinan Kaya if (!mdesc) 40367a2003eSSinan Kaya return NULL; 40467a2003eSSinan Kaya 40567a2003eSSinan Kaya hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, 40667a2003eSSinan Kaya src, dest, len, flags); 40767a2003eSSinan Kaya 40867a2003eSSinan Kaya /* Place descriptor in prepared list */ 40967a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 41067a2003eSSinan Kaya list_add_tail(&mdesc->node, &mchan->prepared); 41167a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 41267a2003eSSinan Kaya 41367a2003eSSinan Kaya return &mdesc->desc; 41467a2003eSSinan Kaya } 41567a2003eSSinan Kaya 41667a2003eSSinan Kaya static int hidma_terminate_channel(struct dma_chan *chan) 41767a2003eSSinan Kaya { 41867a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(chan); 41967a2003eSSinan Kaya struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); 42067a2003eSSinan Kaya struct hidma_desc *tmp, *mdesc; 42167a2003eSSinan Kaya unsigned long irqflags; 42267a2003eSSinan Kaya LIST_HEAD(list); 42367a2003eSSinan Kaya int rc; 42467a2003eSSinan Kaya 42567a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 42667a2003eSSinan Kaya /* give completed requests a chance to finish */ 42767a2003eSSinan Kaya hidma_process_completed(mchan); 42867a2003eSSinan Kaya 42967a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 430793ae66cSSinan Kaya mchan->last_success = 0; 43167a2003eSSinan Kaya list_splice_init(&mchan->active, &list); 43267a2003eSSinan Kaya list_splice_init(&mchan->prepared, &list); 43367a2003eSSinan Kaya list_splice_init(&mchan->completed, &list); 43467a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 43567a2003eSSinan Kaya 43667a2003eSSinan Kaya /* this suspends the existing transfer */ 437d1615ca2SSinan Kaya rc = hidma_ll_disable(dmadev->lldev); 43867a2003eSSinan Kaya if (rc) { 43967a2003eSSinan Kaya dev_err(dmadev->ddev.dev, "channel did not pause\n"); 44067a2003eSSinan Kaya goto out; 44167a2003eSSinan Kaya } 44267a2003eSSinan Kaya 44367a2003eSSinan Kaya /* return all user requests */ 44467a2003eSSinan Kaya list_for_each_entry_safe(mdesc, tmp, &list, node) { 44567a2003eSSinan Kaya struct dma_async_tx_descriptor *txd = &mdesc->desc; 44667a2003eSSinan Kaya 44767a2003eSSinan Kaya dma_descriptor_unmap(txd); 4485ade6683SDave Jiang dmaengine_desc_get_callback_invoke(txd, NULL); 44967a2003eSSinan Kaya dma_run_dependencies(txd); 45067a2003eSSinan Kaya 45167a2003eSSinan Kaya /* move myself to free_list */ 45267a2003eSSinan Kaya list_move(&mdesc->node, &mchan->free); 45367a2003eSSinan Kaya } 45467a2003eSSinan Kaya 455d1615ca2SSinan Kaya rc = hidma_ll_enable(dmadev->lldev); 45667a2003eSSinan Kaya out: 45767a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 45867a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 45967a2003eSSinan Kaya return rc; 46067a2003eSSinan Kaya } 46167a2003eSSinan Kaya 46267a2003eSSinan Kaya static int hidma_terminate_all(struct dma_chan *chan) 46367a2003eSSinan Kaya { 46467a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(chan); 46567a2003eSSinan Kaya struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); 46667a2003eSSinan Kaya int rc; 46767a2003eSSinan Kaya 46867a2003eSSinan Kaya rc = hidma_terminate_channel(chan); 46967a2003eSSinan Kaya if (rc) 47067a2003eSSinan Kaya return rc; 47167a2003eSSinan Kaya 47267a2003eSSinan Kaya /* reinitialize the hardware */ 47367a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 47467a2003eSSinan Kaya rc = hidma_ll_setup(dmadev->lldev); 47567a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 47667a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 47767a2003eSSinan Kaya return rc; 47867a2003eSSinan Kaya } 47967a2003eSSinan Kaya 48067a2003eSSinan Kaya static void hidma_free_chan_resources(struct dma_chan *dmach) 48167a2003eSSinan Kaya { 48267a2003eSSinan Kaya struct hidma_chan *mchan = to_hidma_chan(dmach); 48367a2003eSSinan Kaya struct hidma_dev *mdma = mchan->dmadev; 48467a2003eSSinan Kaya struct hidma_desc *mdesc, *tmp; 48567a2003eSSinan Kaya unsigned long irqflags; 48667a2003eSSinan Kaya LIST_HEAD(descs); 48767a2003eSSinan Kaya 48867a2003eSSinan Kaya /* terminate running transactions and free descriptors */ 48967a2003eSSinan Kaya hidma_terminate_channel(dmach); 49067a2003eSSinan Kaya 49167a2003eSSinan Kaya spin_lock_irqsave(&mchan->lock, irqflags); 49267a2003eSSinan Kaya 49367a2003eSSinan Kaya /* Move data */ 49467a2003eSSinan Kaya list_splice_tail_init(&mchan->free, &descs); 49567a2003eSSinan Kaya 49667a2003eSSinan Kaya /* Free descriptors */ 49767a2003eSSinan Kaya list_for_each_entry_safe(mdesc, tmp, &descs, node) { 49867a2003eSSinan Kaya hidma_ll_free(mdma->lldev, mdesc->tre_ch); 49967a2003eSSinan Kaya list_del(&mdesc->node); 50067a2003eSSinan Kaya kfree(mdesc); 50167a2003eSSinan Kaya } 50267a2003eSSinan Kaya 50367a2003eSSinan Kaya mchan->allocated = 0; 50467a2003eSSinan Kaya spin_unlock_irqrestore(&mchan->lock, irqflags); 50567a2003eSSinan Kaya } 50667a2003eSSinan Kaya 50767a2003eSSinan Kaya static int hidma_pause(struct dma_chan *chan) 50867a2003eSSinan Kaya { 50967a2003eSSinan Kaya struct hidma_chan *mchan; 51067a2003eSSinan Kaya struct hidma_dev *dmadev; 51167a2003eSSinan Kaya 51267a2003eSSinan Kaya mchan = to_hidma_chan(chan); 51367a2003eSSinan Kaya dmadev = to_hidma_dev(mchan->chan.device); 51467a2003eSSinan Kaya if (!mchan->paused) { 51567a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 516d1615ca2SSinan Kaya if (hidma_ll_disable(dmadev->lldev)) 51767a2003eSSinan Kaya dev_warn(dmadev->ddev.dev, "channel did not stop\n"); 51867a2003eSSinan Kaya mchan->paused = true; 51967a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 52067a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 52167a2003eSSinan Kaya } 52267a2003eSSinan Kaya return 0; 52367a2003eSSinan Kaya } 52467a2003eSSinan Kaya 52567a2003eSSinan Kaya static int hidma_resume(struct dma_chan *chan) 52667a2003eSSinan Kaya { 52767a2003eSSinan Kaya struct hidma_chan *mchan; 52867a2003eSSinan Kaya struct hidma_dev *dmadev; 52967a2003eSSinan Kaya int rc = 0; 53067a2003eSSinan Kaya 53167a2003eSSinan Kaya mchan = to_hidma_chan(chan); 53267a2003eSSinan Kaya dmadev = to_hidma_dev(mchan->chan.device); 53367a2003eSSinan Kaya if (mchan->paused) { 53467a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 535d1615ca2SSinan Kaya rc = hidma_ll_enable(dmadev->lldev); 53667a2003eSSinan Kaya if (!rc) 53767a2003eSSinan Kaya mchan->paused = false; 53867a2003eSSinan Kaya else 53967a2003eSSinan Kaya dev_err(dmadev->ddev.dev, 54067a2003eSSinan Kaya "failed to resume the channel"); 54167a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 54267a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 54367a2003eSSinan Kaya } 54467a2003eSSinan Kaya return rc; 54567a2003eSSinan Kaya } 54667a2003eSSinan Kaya 54767a2003eSSinan Kaya static irqreturn_t hidma_chirq_handler(int chirq, void *arg) 54867a2003eSSinan Kaya { 54967a2003eSSinan Kaya struct hidma_lldev *lldev = arg; 55067a2003eSSinan Kaya 55167a2003eSSinan Kaya /* 55267a2003eSSinan Kaya * All interrupts are request driven. 55367a2003eSSinan Kaya * HW doesn't send an interrupt by itself. 55467a2003eSSinan Kaya */ 55567a2003eSSinan Kaya return hidma_ll_inthandler(chirq, lldev); 55667a2003eSSinan Kaya } 55767a2003eSSinan Kaya 558*1c0e3e82SSinan Kaya static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg) 559*1c0e3e82SSinan Kaya { 560*1c0e3e82SSinan Kaya struct hidma_lldev **lldevp = arg; 561*1c0e3e82SSinan Kaya struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp); 562*1c0e3e82SSinan Kaya 563*1c0e3e82SSinan Kaya return hidma_ll_inthandler_msi(chirq, *lldevp, 564*1c0e3e82SSinan Kaya 1 << (chirq - dmadev->msi_virqbase)); 565*1c0e3e82SSinan Kaya } 566*1c0e3e82SSinan Kaya 56742d236f8SSinan Kaya static ssize_t hidma_show_values(struct device *dev, 56842d236f8SSinan Kaya struct device_attribute *attr, char *buf) 56942d236f8SSinan Kaya { 57042d236f8SSinan Kaya struct platform_device *pdev = to_platform_device(dev); 57142d236f8SSinan Kaya struct hidma_dev *mdev = platform_get_drvdata(pdev); 57242d236f8SSinan Kaya 57342d236f8SSinan Kaya buf[0] = 0; 57442d236f8SSinan Kaya 57542d236f8SSinan Kaya if (strcmp(attr->attr.name, "chid") == 0) 57642d236f8SSinan Kaya sprintf(buf, "%d\n", mdev->chidx); 57742d236f8SSinan Kaya 57842d236f8SSinan Kaya return strlen(buf); 57942d236f8SSinan Kaya } 58042d236f8SSinan Kaya 58142d236f8SSinan Kaya static int hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, 58242d236f8SSinan Kaya int mode) 58342d236f8SSinan Kaya { 58442d236f8SSinan Kaya struct device_attribute *attrs; 58542d236f8SSinan Kaya char *name_copy; 58642d236f8SSinan Kaya 58742d236f8SSinan Kaya attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), 58842d236f8SSinan Kaya GFP_KERNEL); 58942d236f8SSinan Kaya if (!attrs) 59042d236f8SSinan Kaya return -ENOMEM; 59142d236f8SSinan Kaya 59242d236f8SSinan Kaya name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); 59342d236f8SSinan Kaya if (!name_copy) 59442d236f8SSinan Kaya return -ENOMEM; 59542d236f8SSinan Kaya 59642d236f8SSinan Kaya attrs->attr.name = name_copy; 59742d236f8SSinan Kaya attrs->attr.mode = mode; 59842d236f8SSinan Kaya attrs->show = hidma_show_values; 59942d236f8SSinan Kaya sysfs_attr_init(&attrs->attr); 60042d236f8SSinan Kaya 60142d236f8SSinan Kaya return device_create_file(dev->ddev.dev, attrs); 60242d236f8SSinan Kaya } 60342d236f8SSinan Kaya 604*1c0e3e82SSinan Kaya #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 605*1c0e3e82SSinan Kaya static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) 606*1c0e3e82SSinan Kaya { 607*1c0e3e82SSinan Kaya struct device *dev = msi_desc_to_dev(desc); 608*1c0e3e82SSinan Kaya struct hidma_dev *dmadev = dev_get_drvdata(dev); 609*1c0e3e82SSinan Kaya 610*1c0e3e82SSinan Kaya if (!desc->platform.msi_index) { 611*1c0e3e82SSinan Kaya writel(msg->address_lo, dmadev->dev_evca + 0x118); 612*1c0e3e82SSinan Kaya writel(msg->address_hi, dmadev->dev_evca + 0x11C); 613*1c0e3e82SSinan Kaya writel(msg->data, dmadev->dev_evca + 0x120); 614*1c0e3e82SSinan Kaya } 615*1c0e3e82SSinan Kaya } 616*1c0e3e82SSinan Kaya #endif 617*1c0e3e82SSinan Kaya 618*1c0e3e82SSinan Kaya static void hidma_free_msis(struct hidma_dev *dmadev) 619*1c0e3e82SSinan Kaya { 620*1c0e3e82SSinan Kaya #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 621*1c0e3e82SSinan Kaya struct device *dev = dmadev->ddev.dev; 622*1c0e3e82SSinan Kaya struct msi_desc *desc; 623*1c0e3e82SSinan Kaya 624*1c0e3e82SSinan Kaya /* free allocated MSI interrupts above */ 625*1c0e3e82SSinan Kaya for_each_msi_entry(desc, dev) 626*1c0e3e82SSinan Kaya devm_free_irq(dev, desc->irq, &dmadev->lldev); 627*1c0e3e82SSinan Kaya 628*1c0e3e82SSinan Kaya platform_msi_domain_free_irqs(dev); 629*1c0e3e82SSinan Kaya #endif 630*1c0e3e82SSinan Kaya } 631*1c0e3e82SSinan Kaya 632*1c0e3e82SSinan Kaya static int hidma_request_msi(struct hidma_dev *dmadev, 633*1c0e3e82SSinan Kaya struct platform_device *pdev) 634*1c0e3e82SSinan Kaya { 635*1c0e3e82SSinan Kaya #ifdef CONFIG_GENERIC_MSI_IRQ_DOMAIN 636*1c0e3e82SSinan Kaya int rc; 637*1c0e3e82SSinan Kaya struct msi_desc *desc; 638*1c0e3e82SSinan Kaya struct msi_desc *failed_desc = NULL; 639*1c0e3e82SSinan Kaya 640*1c0e3e82SSinan Kaya rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, 641*1c0e3e82SSinan Kaya hidma_write_msi_msg); 642*1c0e3e82SSinan Kaya if (rc) 643*1c0e3e82SSinan Kaya return rc; 644*1c0e3e82SSinan Kaya 645*1c0e3e82SSinan Kaya for_each_msi_entry(desc, &pdev->dev) { 646*1c0e3e82SSinan Kaya if (!desc->platform.msi_index) 647*1c0e3e82SSinan Kaya dmadev->msi_virqbase = desc->irq; 648*1c0e3e82SSinan Kaya 649*1c0e3e82SSinan Kaya rc = devm_request_irq(&pdev->dev, desc->irq, 650*1c0e3e82SSinan Kaya hidma_chirq_handler_msi, 651*1c0e3e82SSinan Kaya 0, "qcom-hidma-msi", 652*1c0e3e82SSinan Kaya &dmadev->lldev); 653*1c0e3e82SSinan Kaya if (rc) { 654*1c0e3e82SSinan Kaya failed_desc = desc; 655*1c0e3e82SSinan Kaya break; 656*1c0e3e82SSinan Kaya } 657*1c0e3e82SSinan Kaya } 658*1c0e3e82SSinan Kaya 659*1c0e3e82SSinan Kaya if (rc) { 660*1c0e3e82SSinan Kaya /* free allocated MSI interrupts above */ 661*1c0e3e82SSinan Kaya for_each_msi_entry(desc, &pdev->dev) { 662*1c0e3e82SSinan Kaya if (desc == failed_desc) 663*1c0e3e82SSinan Kaya break; 664*1c0e3e82SSinan Kaya devm_free_irq(&pdev->dev, desc->irq, 665*1c0e3e82SSinan Kaya &dmadev->lldev); 666*1c0e3e82SSinan Kaya } 667*1c0e3e82SSinan Kaya } else { 668*1c0e3e82SSinan Kaya /* Add callback to free MSIs on teardown */ 669*1c0e3e82SSinan Kaya hidma_ll_setup_irq(dmadev->lldev, true); 670*1c0e3e82SSinan Kaya 671*1c0e3e82SSinan Kaya } 672*1c0e3e82SSinan Kaya if (rc) 673*1c0e3e82SSinan Kaya dev_warn(&pdev->dev, 674*1c0e3e82SSinan Kaya "failed to request MSI irq, falling back to wired IRQ\n"); 675*1c0e3e82SSinan Kaya return rc; 676*1c0e3e82SSinan Kaya #else 677*1c0e3e82SSinan Kaya return -EINVAL; 678*1c0e3e82SSinan Kaya #endif 679*1c0e3e82SSinan Kaya } 680*1c0e3e82SSinan Kaya 681*1c0e3e82SSinan Kaya static bool hidma_msi_capable(struct device *dev) 682*1c0e3e82SSinan Kaya { 683*1c0e3e82SSinan Kaya struct acpi_device *adev = ACPI_COMPANION(dev); 684*1c0e3e82SSinan Kaya const char *of_compat; 685*1c0e3e82SSinan Kaya int ret = -EINVAL; 686*1c0e3e82SSinan Kaya 687*1c0e3e82SSinan Kaya if (!adev || acpi_disabled) { 688*1c0e3e82SSinan Kaya ret = device_property_read_string(dev, "compatible", 689*1c0e3e82SSinan Kaya &of_compat); 690*1c0e3e82SSinan Kaya if (ret) 691*1c0e3e82SSinan Kaya return false; 692*1c0e3e82SSinan Kaya 693*1c0e3e82SSinan Kaya ret = strcmp(of_compat, "qcom,hidma-1.1"); 694*1c0e3e82SSinan Kaya } else { 695*1c0e3e82SSinan Kaya #ifdef CONFIG_ACPI 696*1c0e3e82SSinan Kaya ret = strcmp(acpi_device_hid(adev), "QCOM8062"); 697*1c0e3e82SSinan Kaya #endif 698*1c0e3e82SSinan Kaya } 699*1c0e3e82SSinan Kaya return ret == 0; 700*1c0e3e82SSinan Kaya } 701*1c0e3e82SSinan Kaya 70267a2003eSSinan Kaya static int hidma_probe(struct platform_device *pdev) 70367a2003eSSinan Kaya { 70467a2003eSSinan Kaya struct hidma_dev *dmadev; 70567a2003eSSinan Kaya struct resource *trca_resource; 70667a2003eSSinan Kaya struct resource *evca_resource; 70767a2003eSSinan Kaya int chirq; 70867a2003eSSinan Kaya void __iomem *evca; 70967a2003eSSinan Kaya void __iomem *trca; 71067a2003eSSinan Kaya int rc; 711*1c0e3e82SSinan Kaya bool msi; 71267a2003eSSinan Kaya 71367a2003eSSinan Kaya pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); 71467a2003eSSinan Kaya pm_runtime_use_autosuspend(&pdev->dev); 71567a2003eSSinan Kaya pm_runtime_set_active(&pdev->dev); 71667a2003eSSinan Kaya pm_runtime_enable(&pdev->dev); 71767a2003eSSinan Kaya 71867a2003eSSinan Kaya trca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 0); 71967a2003eSSinan Kaya trca = devm_ioremap_resource(&pdev->dev, trca_resource); 72067a2003eSSinan Kaya if (IS_ERR(trca)) { 72167a2003eSSinan Kaya rc = -ENOMEM; 72267a2003eSSinan Kaya goto bailout; 72367a2003eSSinan Kaya } 72467a2003eSSinan Kaya 72567a2003eSSinan Kaya evca_resource = platform_get_resource(pdev, IORESOURCE_MEM, 1); 72667a2003eSSinan Kaya evca = devm_ioremap_resource(&pdev->dev, evca_resource); 72767a2003eSSinan Kaya if (IS_ERR(evca)) { 72867a2003eSSinan Kaya rc = -ENOMEM; 72967a2003eSSinan Kaya goto bailout; 73067a2003eSSinan Kaya } 73167a2003eSSinan Kaya 73267a2003eSSinan Kaya /* 73367a2003eSSinan Kaya * This driver only handles the channel IRQs. 73467a2003eSSinan Kaya * Common IRQ is handled by the management driver. 73567a2003eSSinan Kaya */ 73667a2003eSSinan Kaya chirq = platform_get_irq(pdev, 0); 73767a2003eSSinan Kaya if (chirq < 0) { 73867a2003eSSinan Kaya rc = -ENODEV; 73967a2003eSSinan Kaya goto bailout; 74067a2003eSSinan Kaya } 74167a2003eSSinan Kaya 74267a2003eSSinan Kaya dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); 74367a2003eSSinan Kaya if (!dmadev) { 74467a2003eSSinan Kaya rc = -ENOMEM; 74567a2003eSSinan Kaya goto bailout; 74667a2003eSSinan Kaya } 74767a2003eSSinan Kaya 74867a2003eSSinan Kaya INIT_LIST_HEAD(&dmadev->ddev.channels); 74967a2003eSSinan Kaya spin_lock_init(&dmadev->lock); 75067a2003eSSinan Kaya dmadev->ddev.dev = &pdev->dev; 75167a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 75267a2003eSSinan Kaya 75367a2003eSSinan Kaya dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); 75467a2003eSSinan Kaya if (WARN_ON(!pdev->dev.dma_mask)) { 75567a2003eSSinan Kaya rc = -ENXIO; 75667a2003eSSinan Kaya goto dmafree; 75767a2003eSSinan Kaya } 75867a2003eSSinan Kaya 75967a2003eSSinan Kaya dmadev->dev_evca = evca; 76067a2003eSSinan Kaya dmadev->evca_resource = evca_resource; 76167a2003eSSinan Kaya dmadev->dev_trca = trca; 76267a2003eSSinan Kaya dmadev->trca_resource = trca_resource; 76367a2003eSSinan Kaya dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; 76467a2003eSSinan Kaya dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; 76567a2003eSSinan Kaya dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; 76667a2003eSSinan Kaya dmadev->ddev.device_tx_status = hidma_tx_status; 76767a2003eSSinan Kaya dmadev->ddev.device_issue_pending = hidma_issue_pending; 76867a2003eSSinan Kaya dmadev->ddev.device_pause = hidma_pause; 76967a2003eSSinan Kaya dmadev->ddev.device_resume = hidma_resume; 77067a2003eSSinan Kaya dmadev->ddev.device_terminate_all = hidma_terminate_all; 77167a2003eSSinan Kaya dmadev->ddev.copy_align = 8; 77267a2003eSSinan Kaya 773*1c0e3e82SSinan Kaya /* 774*1c0e3e82SSinan Kaya * Determine the MSI capability of the platform. Old HW doesn't 775*1c0e3e82SSinan Kaya * support MSI. 776*1c0e3e82SSinan Kaya */ 777*1c0e3e82SSinan Kaya msi = hidma_msi_capable(&pdev->dev); 778*1c0e3e82SSinan Kaya 77967a2003eSSinan Kaya device_property_read_u32(&pdev->dev, "desc-count", 78067a2003eSSinan Kaya &dmadev->nr_descriptors); 78167a2003eSSinan Kaya 78267a2003eSSinan Kaya if (!dmadev->nr_descriptors && nr_desc_prm) 78367a2003eSSinan Kaya dmadev->nr_descriptors = nr_desc_prm; 78467a2003eSSinan Kaya 78567a2003eSSinan Kaya if (!dmadev->nr_descriptors) 78667a2003eSSinan Kaya dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; 78767a2003eSSinan Kaya 78867a2003eSSinan Kaya dmadev->chidx = readl(dmadev->dev_trca + 0x28); 78967a2003eSSinan Kaya 79067a2003eSSinan Kaya /* Set DMA mask to 64 bits. */ 79167a2003eSSinan Kaya rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); 79267a2003eSSinan Kaya if (rc) { 79367a2003eSSinan Kaya dev_warn(&pdev->dev, "unable to set coherent mask to 64"); 79467a2003eSSinan Kaya rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); 79567a2003eSSinan Kaya if (rc) 79667a2003eSSinan Kaya goto dmafree; 79767a2003eSSinan Kaya } 79867a2003eSSinan Kaya 79967a2003eSSinan Kaya dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, 80067a2003eSSinan Kaya dmadev->nr_descriptors, dmadev->dev_trca, 80167a2003eSSinan Kaya dmadev->dev_evca, dmadev->chidx); 80267a2003eSSinan Kaya if (!dmadev->lldev) { 80367a2003eSSinan Kaya rc = -EPROBE_DEFER; 80467a2003eSSinan Kaya goto dmafree; 80567a2003eSSinan Kaya } 80667a2003eSSinan Kaya 807*1c0e3e82SSinan Kaya platform_set_drvdata(pdev, dmadev); 808*1c0e3e82SSinan Kaya if (msi) 809*1c0e3e82SSinan Kaya rc = hidma_request_msi(dmadev, pdev); 810*1c0e3e82SSinan Kaya 811*1c0e3e82SSinan Kaya if (!msi || rc) { 812*1c0e3e82SSinan Kaya hidma_ll_setup_irq(dmadev->lldev, false); 813*1c0e3e82SSinan Kaya rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, 814*1c0e3e82SSinan Kaya 0, "qcom-hidma", dmadev->lldev); 81567a2003eSSinan Kaya if (rc) 81667a2003eSSinan Kaya goto uninit; 817*1c0e3e82SSinan Kaya } 81867a2003eSSinan Kaya 81967a2003eSSinan Kaya INIT_LIST_HEAD(&dmadev->ddev.channels); 82067a2003eSSinan Kaya rc = hidma_chan_init(dmadev, 0); 82167a2003eSSinan Kaya if (rc) 82267a2003eSSinan Kaya goto uninit; 82367a2003eSSinan Kaya 82467a2003eSSinan Kaya rc = dma_async_device_register(&dmadev->ddev); 82567a2003eSSinan Kaya if (rc) 82667a2003eSSinan Kaya goto uninit; 82767a2003eSSinan Kaya 82867a2003eSSinan Kaya dmadev->irq = chirq; 82967a2003eSSinan Kaya tasklet_init(&dmadev->task, hidma_issue_task, (unsigned long)dmadev); 830570d0176SSinan Kaya hidma_debug_init(dmadev); 83142d236f8SSinan Kaya hidma_create_sysfs_entry(dmadev, "chid", S_IRUGO); 83267a2003eSSinan Kaya dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); 83367a2003eSSinan Kaya pm_runtime_mark_last_busy(dmadev->ddev.dev); 83467a2003eSSinan Kaya pm_runtime_put_autosuspend(dmadev->ddev.dev); 83567a2003eSSinan Kaya return 0; 83667a2003eSSinan Kaya 83767a2003eSSinan Kaya uninit: 838*1c0e3e82SSinan Kaya if (msi) 839*1c0e3e82SSinan Kaya hidma_free_msis(dmadev); 840*1c0e3e82SSinan Kaya 841570d0176SSinan Kaya hidma_debug_uninit(dmadev); 84267a2003eSSinan Kaya hidma_ll_uninit(dmadev->lldev); 84367a2003eSSinan Kaya dmafree: 84467a2003eSSinan Kaya if (dmadev) 84567a2003eSSinan Kaya hidma_free(dmadev); 84667a2003eSSinan Kaya bailout: 84767a2003eSSinan Kaya pm_runtime_put_sync(&pdev->dev); 84867a2003eSSinan Kaya pm_runtime_disable(&pdev->dev); 84967a2003eSSinan Kaya return rc; 85067a2003eSSinan Kaya } 85167a2003eSSinan Kaya 85267a2003eSSinan Kaya static int hidma_remove(struct platform_device *pdev) 85367a2003eSSinan Kaya { 85467a2003eSSinan Kaya struct hidma_dev *dmadev = platform_get_drvdata(pdev); 85567a2003eSSinan Kaya 85667a2003eSSinan Kaya pm_runtime_get_sync(dmadev->ddev.dev); 85767a2003eSSinan Kaya dma_async_device_unregister(&dmadev->ddev); 858*1c0e3e82SSinan Kaya if (!dmadev->lldev->msi_support) 85967a2003eSSinan Kaya devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); 860*1c0e3e82SSinan Kaya else 861*1c0e3e82SSinan Kaya hidma_free_msis(dmadev); 862*1c0e3e82SSinan Kaya 863bd16934aSVinod Koul tasklet_kill(&dmadev->task); 864570d0176SSinan Kaya hidma_debug_uninit(dmadev); 86567a2003eSSinan Kaya hidma_ll_uninit(dmadev->lldev); 86667a2003eSSinan Kaya hidma_free(dmadev); 86767a2003eSSinan Kaya 86867a2003eSSinan Kaya dev_info(&pdev->dev, "HI-DMA engine removed\n"); 86967a2003eSSinan Kaya pm_runtime_put_sync_suspend(&pdev->dev); 87067a2003eSSinan Kaya pm_runtime_disable(&pdev->dev); 87167a2003eSSinan Kaya 87267a2003eSSinan Kaya return 0; 87367a2003eSSinan Kaya } 87467a2003eSSinan Kaya 87567a2003eSSinan Kaya #if IS_ENABLED(CONFIG_ACPI) 87667a2003eSSinan Kaya static const struct acpi_device_id hidma_acpi_ids[] = { 87767a2003eSSinan Kaya {"QCOM8061"}, 878*1c0e3e82SSinan Kaya {"QCOM8062"}, 87967a2003eSSinan Kaya {}, 88067a2003eSSinan Kaya }; 88167a2003eSSinan Kaya #endif 88267a2003eSSinan Kaya 88367a2003eSSinan Kaya static const struct of_device_id hidma_match[] = { 88467a2003eSSinan Kaya {.compatible = "qcom,hidma-1.0",}, 885*1c0e3e82SSinan Kaya {.compatible = "qcom,hidma-1.1",}, 88667a2003eSSinan Kaya {}, 88767a2003eSSinan Kaya }; 88867a2003eSSinan Kaya MODULE_DEVICE_TABLE(of, hidma_match); 88967a2003eSSinan Kaya 89067a2003eSSinan Kaya static struct platform_driver hidma_driver = { 89167a2003eSSinan Kaya .probe = hidma_probe, 89267a2003eSSinan Kaya .remove = hidma_remove, 89367a2003eSSinan Kaya .driver = { 89467a2003eSSinan Kaya .name = "hidma", 89567a2003eSSinan Kaya .of_match_table = hidma_match, 89667a2003eSSinan Kaya .acpi_match_table = ACPI_PTR(hidma_acpi_ids), 89767a2003eSSinan Kaya }, 89867a2003eSSinan Kaya }; 89967a2003eSSinan Kaya 90067a2003eSSinan Kaya module_platform_driver(hidma_driver); 90167a2003eSSinan Kaya MODULE_LICENSE("GPL v2"); 902