1 /*
2 * Qualcomm Technologies HIDMA DMA engine interface
3 *
4 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
5 *
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 and
8 * only version 2 as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope that it will be useful,
11 * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 * GNU General Public License for more details.
14 */
15
16 /*
17 * Copyright (C) Freescale Semicondutor, Inc. 2007, 2008.
18 * Copyright (C) Semihalf 2009
19 * Copyright (C) Ilya Yanok, Emcraft Systems 2010
20 * Copyright (C) Alexander Popov, Promcontroller 2014
21 *
22 * Written by Piotr Ziecik <kosmo@semihalf.com>. Hardware description
23 * (defines, structures and comments) was taken from MPC5121 DMA driver
24 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
25 *
26 * Approved as OSADL project by a majority of OSADL members and funded
27 * by OSADL membership fees in 2009; for details see www.osadl.org.
28 *
29 * This program is free software; you can redistribute it and/or modify it
30 * under the terms of the GNU General Public License as published by the Free
31 * Software Foundation; either version 2 of the License, or (at your option)
32 * any later version.
33 *
34 * This program is distributed in the hope that it will be useful, but WITHOUT
35 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
36 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
37 * more details.
38 *
39 * The full GNU General Public License is included in this distribution in the
40 * file called COPYING.
41 */
42
43 /* Linux Foundation elects GPLv2 license only. */
44
45 #include <linux/dmaengine.h>
46 #include <linux/dma-mapping.h>
47 #include <linux/list.h>
48 #include <linux/module.h>
49 #include <linux/platform_device.h>
50 #include <linux/slab.h>
51 #include <linux/spinlock.h>
52 #include <linux/property.h>
53 #include <linux/delay.h>
54 #include <linux/acpi.h>
55 #include <linux/irq.h>
56 #include <linux/atomic.h>
57 #include <linux/pm_runtime.h>
58 #include <linux/msi.h>
59
60 #include "../dmaengine.h"
61 #include "hidma.h"
62
63 /*
64 * Default idle time is 2 seconds. This parameter can
65 * be overridden by changing the following
66 * /sys/bus/platform/devices/QCOM8061:<xy>/power/autosuspend_delay_ms
67 * during kernel boot.
68 */
69 #define HIDMA_AUTOSUSPEND_TIMEOUT 2000
70 #define HIDMA_ERR_INFO_SW 0xFF
71 #define HIDMA_ERR_CODE_UNEXPECTED_TERMINATE 0x0
72 #define HIDMA_NR_DEFAULT_DESC 10
73 #define HIDMA_MSI_INTS 11
74
to_hidma_dev(struct dma_device * dmadev)75 static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev)
76 {
77 return container_of(dmadev, struct hidma_dev, ddev);
78 }
79
80 static inline
to_hidma_dev_from_lldev(struct hidma_lldev ** _lldevp)81 struct hidma_dev *to_hidma_dev_from_lldev(struct hidma_lldev **_lldevp)
82 {
83 return container_of(_lldevp, struct hidma_dev, lldev);
84 }
85
to_hidma_chan(struct dma_chan * dmach)86 static inline struct hidma_chan *to_hidma_chan(struct dma_chan *dmach)
87 {
88 return container_of(dmach, struct hidma_chan, chan);
89 }
90
hidma_free(struct hidma_dev * dmadev)91 static void hidma_free(struct hidma_dev *dmadev)
92 {
93 INIT_LIST_HEAD(&dmadev->ddev.channels);
94 }
95
96 static unsigned int nr_desc_prm;
97 module_param(nr_desc_prm, uint, 0644);
98 MODULE_PARM_DESC(nr_desc_prm, "number of descriptors (default: 0)");
99
100 enum hidma_cap {
101 HIDMA_MSI_CAP = 1,
102 HIDMA_IDENTITY_CAP,
103 };
104
105 /* process completed descriptors */
hidma_process_completed(struct hidma_chan * mchan)106 static void hidma_process_completed(struct hidma_chan *mchan)
107 {
108 struct dma_device *ddev = mchan->chan.device;
109 struct hidma_dev *mdma = to_hidma_dev(ddev);
110 struct dma_async_tx_descriptor *desc;
111 dma_cookie_t last_cookie;
112 struct hidma_desc *mdesc;
113 struct hidma_desc *next;
114 unsigned long irqflags;
115 struct list_head list;
116
117 INIT_LIST_HEAD(&list);
118
119 /* Get all completed descriptors */
120 spin_lock_irqsave(&mchan->lock, irqflags);
121 list_splice_tail_init(&mchan->completed, &list);
122 spin_unlock_irqrestore(&mchan->lock, irqflags);
123
124 /* Execute callbacks and run dependencies */
125 list_for_each_entry_safe(mdesc, next, &list, node) {
126 enum dma_status llstat;
127 struct dmaengine_desc_callback cb;
128 struct dmaengine_result result;
129
130 desc = &mdesc->desc;
131 last_cookie = desc->cookie;
132
133 llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch);
134
135 spin_lock_irqsave(&mchan->lock, irqflags);
136 if (llstat == DMA_COMPLETE) {
137 mchan->last_success = last_cookie;
138 result.result = DMA_TRANS_NOERROR;
139 } else {
140 result.result = DMA_TRANS_ABORTED;
141 }
142
143 dma_cookie_complete(desc);
144 spin_unlock_irqrestore(&mchan->lock, irqflags);
145
146 dmaengine_desc_get_callback(desc, &cb);
147
148 dma_run_dependencies(desc);
149
150 spin_lock_irqsave(&mchan->lock, irqflags);
151 list_move(&mdesc->node, &mchan->free);
152 spin_unlock_irqrestore(&mchan->lock, irqflags);
153
154 dmaengine_desc_callback_invoke(&cb, &result);
155 }
156 }
157
158 /*
159 * Called once for each submitted descriptor.
160 * PM is locked once for each descriptor that is currently
161 * in execution.
162 */
hidma_callback(void * data)163 static void hidma_callback(void *data)
164 {
165 struct hidma_desc *mdesc = data;
166 struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan);
167 struct dma_device *ddev = mchan->chan.device;
168 struct hidma_dev *dmadev = to_hidma_dev(ddev);
169 unsigned long irqflags;
170 bool queued = false;
171
172 spin_lock_irqsave(&mchan->lock, irqflags);
173 if (mdesc->node.next) {
174 /* Delete from the active list, add to completed list */
175 list_move_tail(&mdesc->node, &mchan->completed);
176 queued = true;
177
178 /* calculate the next running descriptor */
179 mchan->running = list_first_entry(&mchan->active,
180 struct hidma_desc, node);
181 }
182 spin_unlock_irqrestore(&mchan->lock, irqflags);
183
184 hidma_process_completed(mchan);
185
186 if (queued) {
187 pm_runtime_mark_last_busy(dmadev->ddev.dev);
188 pm_runtime_put_autosuspend(dmadev->ddev.dev);
189 }
190 }
191
hidma_chan_init(struct hidma_dev * dmadev,u32 dma_sig)192 static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig)
193 {
194 struct hidma_chan *mchan;
195 struct dma_device *ddev;
196
197 mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL);
198 if (!mchan)
199 return -ENOMEM;
200
201 ddev = &dmadev->ddev;
202 mchan->dma_sig = dma_sig;
203 mchan->dmadev = dmadev;
204 mchan->chan.device = ddev;
205 dma_cookie_init(&mchan->chan);
206
207 INIT_LIST_HEAD(&mchan->free);
208 INIT_LIST_HEAD(&mchan->prepared);
209 INIT_LIST_HEAD(&mchan->active);
210 INIT_LIST_HEAD(&mchan->completed);
211 INIT_LIST_HEAD(&mchan->queued);
212
213 spin_lock_init(&mchan->lock);
214 list_add_tail(&mchan->chan.device_node, &ddev->channels);
215 return 0;
216 }
217
hidma_issue_task(struct tasklet_struct * t)218 static void hidma_issue_task(struct tasklet_struct *t)
219 {
220 struct hidma_dev *dmadev = from_tasklet(dmadev, t, task);
221
222 pm_runtime_get_sync(dmadev->ddev.dev);
223 hidma_ll_start(dmadev->lldev);
224 }
225
hidma_issue_pending(struct dma_chan * dmach)226 static void hidma_issue_pending(struct dma_chan *dmach)
227 {
228 struct hidma_chan *mchan = to_hidma_chan(dmach);
229 struct hidma_dev *dmadev = mchan->dmadev;
230 unsigned long flags;
231 struct hidma_desc *qdesc, *next;
232 int status;
233
234 spin_lock_irqsave(&mchan->lock, flags);
235 list_for_each_entry_safe(qdesc, next, &mchan->queued, node) {
236 hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch);
237 list_move_tail(&qdesc->node, &mchan->active);
238 }
239
240 if (!mchan->running) {
241 struct hidma_desc *desc = list_first_entry(&mchan->active,
242 struct hidma_desc,
243 node);
244 mchan->running = desc;
245 }
246 spin_unlock_irqrestore(&mchan->lock, flags);
247
248 /* PM will be released in hidma_callback function. */
249 status = pm_runtime_get(dmadev->ddev.dev);
250 if (status < 0)
251 tasklet_schedule(&dmadev->task);
252 else
253 hidma_ll_start(dmadev->lldev);
254 }
255
hidma_txn_is_success(dma_cookie_t cookie,dma_cookie_t last_success,dma_cookie_t last_used)256 static inline bool hidma_txn_is_success(dma_cookie_t cookie,
257 dma_cookie_t last_success, dma_cookie_t last_used)
258 {
259 if (last_success <= last_used) {
260 if ((cookie <= last_success) || (cookie > last_used))
261 return true;
262 } else {
263 if ((cookie <= last_success) && (cookie > last_used))
264 return true;
265 }
266 return false;
267 }
268
hidma_tx_status(struct dma_chan * dmach,dma_cookie_t cookie,struct dma_tx_state * txstate)269 static enum dma_status hidma_tx_status(struct dma_chan *dmach,
270 dma_cookie_t cookie,
271 struct dma_tx_state *txstate)
272 {
273 struct hidma_chan *mchan = to_hidma_chan(dmach);
274 enum dma_status ret;
275
276 ret = dma_cookie_status(dmach, cookie, txstate);
277 if (ret == DMA_COMPLETE) {
278 bool is_success;
279
280 is_success = hidma_txn_is_success(cookie, mchan->last_success,
281 dmach->cookie);
282 return is_success ? ret : DMA_ERROR;
283 }
284
285 if (mchan->paused && (ret == DMA_IN_PROGRESS)) {
286 unsigned long flags;
287 dma_cookie_t runcookie;
288
289 spin_lock_irqsave(&mchan->lock, flags);
290 if (mchan->running)
291 runcookie = mchan->running->desc.cookie;
292 else
293 runcookie = -EINVAL;
294
295 if (runcookie == cookie)
296 ret = DMA_PAUSED;
297
298 spin_unlock_irqrestore(&mchan->lock, flags);
299 }
300
301 return ret;
302 }
303
304 /*
305 * Submit descriptor to hardware.
306 * Lock the PM for each descriptor we are sending.
307 */
hidma_tx_submit(struct dma_async_tx_descriptor * txd)308 static dma_cookie_t hidma_tx_submit(struct dma_async_tx_descriptor *txd)
309 {
310 struct hidma_chan *mchan = to_hidma_chan(txd->chan);
311 struct hidma_dev *dmadev = mchan->dmadev;
312 struct hidma_desc *mdesc;
313 unsigned long irqflags;
314 dma_cookie_t cookie;
315
316 pm_runtime_get_sync(dmadev->ddev.dev);
317 if (!hidma_ll_isenabled(dmadev->lldev)) {
318 pm_runtime_mark_last_busy(dmadev->ddev.dev);
319 pm_runtime_put_autosuspend(dmadev->ddev.dev);
320 return -ENODEV;
321 }
322 pm_runtime_mark_last_busy(dmadev->ddev.dev);
323 pm_runtime_put_autosuspend(dmadev->ddev.dev);
324
325 mdesc = container_of(txd, struct hidma_desc, desc);
326 spin_lock_irqsave(&mchan->lock, irqflags);
327
328 /* Move descriptor to queued */
329 list_move_tail(&mdesc->node, &mchan->queued);
330
331 /* Update cookie */
332 cookie = dma_cookie_assign(txd);
333
334 spin_unlock_irqrestore(&mchan->lock, irqflags);
335
336 return cookie;
337 }
338
hidma_alloc_chan_resources(struct dma_chan * dmach)339 static int hidma_alloc_chan_resources(struct dma_chan *dmach)
340 {
341 struct hidma_chan *mchan = to_hidma_chan(dmach);
342 struct hidma_dev *dmadev = mchan->dmadev;
343 struct hidma_desc *mdesc, *tmp;
344 unsigned long irqflags;
345 LIST_HEAD(descs);
346 unsigned int i;
347 int rc = 0;
348
349 if (mchan->allocated)
350 return 0;
351
352 /* Alloc descriptors for this channel */
353 for (i = 0; i < dmadev->nr_descriptors; i++) {
354 mdesc = kzalloc_obj(struct hidma_desc, GFP_NOWAIT);
355 if (!mdesc) {
356 rc = -ENOMEM;
357 break;
358 }
359 dma_async_tx_descriptor_init(&mdesc->desc, dmach);
360 mdesc->desc.tx_submit = hidma_tx_submit;
361
362 rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig,
363 "DMA engine", hidma_callback, mdesc,
364 &mdesc->tre_ch);
365 if (rc) {
366 dev_err(dmach->device->dev,
367 "channel alloc failed at %u\n", i);
368 kfree(mdesc);
369 break;
370 }
371 list_add_tail(&mdesc->node, &descs);
372 }
373
374 if (rc) {
375 /* return the allocated descriptors */
376 list_for_each_entry_safe(mdesc, tmp, &descs, node) {
377 hidma_ll_free(dmadev->lldev, mdesc->tre_ch);
378 kfree(mdesc);
379 }
380 return rc;
381 }
382
383 spin_lock_irqsave(&mchan->lock, irqflags);
384 list_splice_tail_init(&descs, &mchan->free);
385 mchan->allocated = true;
386 spin_unlock_irqrestore(&mchan->lock, irqflags);
387 return 1;
388 }
389
390 static struct dma_async_tx_descriptor *
hidma_prep_dma_memcpy(struct dma_chan * dmach,dma_addr_t dest,dma_addr_t src,size_t len,unsigned long flags)391 hidma_prep_dma_memcpy(struct dma_chan *dmach, dma_addr_t dest, dma_addr_t src,
392 size_t len, unsigned long flags)
393 {
394 struct hidma_chan *mchan = to_hidma_chan(dmach);
395 struct hidma_desc *mdesc = NULL;
396 struct hidma_dev *mdma = mchan->dmadev;
397 unsigned long irqflags;
398
399 /* Get free descriptor */
400 spin_lock_irqsave(&mchan->lock, irqflags);
401 if (!list_empty(&mchan->free)) {
402 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
403 list_del(&mdesc->node);
404 }
405 spin_unlock_irqrestore(&mchan->lock, irqflags);
406
407 if (!mdesc)
408 return NULL;
409
410 mdesc->desc.flags = flags;
411 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
412 src, dest, len, flags,
413 HIDMA_TRE_MEMCPY);
414
415 /* Place descriptor in prepared list */
416 spin_lock_irqsave(&mchan->lock, irqflags);
417 list_add_tail(&mdesc->node, &mchan->prepared);
418 spin_unlock_irqrestore(&mchan->lock, irqflags);
419
420 return &mdesc->desc;
421 }
422
423 static struct dma_async_tx_descriptor *
hidma_prep_dma_memset(struct dma_chan * dmach,dma_addr_t dest,int value,size_t len,unsigned long flags)424 hidma_prep_dma_memset(struct dma_chan *dmach, dma_addr_t dest, int value,
425 size_t len, unsigned long flags)
426 {
427 struct hidma_chan *mchan = to_hidma_chan(dmach);
428 struct hidma_desc *mdesc = NULL;
429 struct hidma_dev *mdma = mchan->dmadev;
430 unsigned long irqflags;
431 u64 byte_pattern, fill_pattern;
432
433 /* Get free descriptor */
434 spin_lock_irqsave(&mchan->lock, irqflags);
435 if (!list_empty(&mchan->free)) {
436 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);
437 list_del(&mdesc->node);
438 }
439 spin_unlock_irqrestore(&mchan->lock, irqflags);
440
441 if (!mdesc)
442 return NULL;
443
444 byte_pattern = (char)value;
445 fill_pattern = (byte_pattern << 56) |
446 (byte_pattern << 48) |
447 (byte_pattern << 40) |
448 (byte_pattern << 32) |
449 (byte_pattern << 24) |
450 (byte_pattern << 16) |
451 (byte_pattern << 8) |
452 byte_pattern;
453
454 mdesc->desc.flags = flags;
455 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch,
456 fill_pattern, dest, len, flags,
457 HIDMA_TRE_MEMSET);
458
459 /* Place descriptor in prepared list */
460 spin_lock_irqsave(&mchan->lock, irqflags);
461 list_add_tail(&mdesc->node, &mchan->prepared);
462 spin_unlock_irqrestore(&mchan->lock, irqflags);
463
464 return &mdesc->desc;
465 }
466
hidma_terminate_channel(struct dma_chan * chan)467 static int hidma_terminate_channel(struct dma_chan *chan)
468 {
469 struct hidma_chan *mchan = to_hidma_chan(chan);
470 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);
471 struct hidma_desc *tmp, *mdesc;
472 unsigned long irqflags;
473 LIST_HEAD(list);
474 int rc;
475
476 pm_runtime_get_sync(dmadev->ddev.dev);
477 /* give completed requests a chance to finish */
478 hidma_process_completed(mchan);
479
480 spin_lock_irqsave(&mchan->lock, irqflags);
481 mchan->last_success = 0;
482 list_splice_init(&mchan->active, &list);
483 list_splice_init(&mchan->prepared, &list);
484 list_splice_init(&mchan->completed, &list);
485 list_splice_init(&mchan->queued, &list);
486 spin_unlock_irqrestore(&mchan->lock, irqflags);
487
488 /* this suspends the existing transfer */
489 rc = hidma_ll_disable(dmadev->lldev);
490 if (rc) {
491 dev_err(dmadev->ddev.dev, "channel did not pause\n");
492 goto out;
493 }
494
495 /* return all user requests */
496 list_for_each_entry_safe(mdesc, tmp, &list, node) {
497 struct dma_async_tx_descriptor *txd = &mdesc->desc;
498
499 dma_descriptor_unmap(txd);
500 dmaengine_desc_get_callback_invoke(txd, NULL);
501 dma_run_dependencies(txd);
502
503 /* move myself to free_list */
504 list_move(&mdesc->node, &mchan->free);
505 }
506
507 rc = hidma_ll_enable(dmadev->lldev);
508 out:
509 pm_runtime_mark_last_busy(dmadev->ddev.dev);
510 pm_runtime_put_autosuspend(dmadev->ddev.dev);
511 return rc;
512 }
513
hidma_terminate_all(struct dma_chan * chan)514 static int hidma_terminate_all(struct dma_chan *chan)
515 {
516 struct hidma_chan *mchan = to_hidma_chan(chan);
517 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);
518 int rc;
519
520 rc = hidma_terminate_channel(chan);
521 if (rc)
522 return rc;
523
524 /* reinitialize the hardware */
525 pm_runtime_get_sync(dmadev->ddev.dev);
526 rc = hidma_ll_setup(dmadev->lldev);
527 pm_runtime_mark_last_busy(dmadev->ddev.dev);
528 pm_runtime_put_autosuspend(dmadev->ddev.dev);
529 return rc;
530 }
531
hidma_free_chan_resources(struct dma_chan * dmach)532 static void hidma_free_chan_resources(struct dma_chan *dmach)
533 {
534 struct hidma_chan *mchan = to_hidma_chan(dmach);
535 struct hidma_dev *mdma = mchan->dmadev;
536 struct hidma_desc *mdesc, *tmp;
537 unsigned long irqflags;
538 LIST_HEAD(descs);
539
540 /* terminate running transactions and free descriptors */
541 hidma_terminate_channel(dmach);
542
543 spin_lock_irqsave(&mchan->lock, irqflags);
544
545 /* Move data */
546 list_splice_tail_init(&mchan->free, &descs);
547
548 /* Free descriptors */
549 list_for_each_entry_safe(mdesc, tmp, &descs, node) {
550 hidma_ll_free(mdma->lldev, mdesc->tre_ch);
551 list_del(&mdesc->node);
552 kfree(mdesc);
553 }
554
555 mchan->allocated = false;
556 spin_unlock_irqrestore(&mchan->lock, irqflags);
557 }
558
hidma_pause(struct dma_chan * chan)559 static int hidma_pause(struct dma_chan *chan)
560 {
561 struct hidma_chan *mchan;
562 struct hidma_dev *dmadev;
563
564 mchan = to_hidma_chan(chan);
565 dmadev = to_hidma_dev(mchan->chan.device);
566 if (!mchan->paused) {
567 pm_runtime_get_sync(dmadev->ddev.dev);
568 if (hidma_ll_disable(dmadev->lldev))
569 dev_warn(dmadev->ddev.dev, "channel did not stop\n");
570 mchan->paused = true;
571 pm_runtime_mark_last_busy(dmadev->ddev.dev);
572 pm_runtime_put_autosuspend(dmadev->ddev.dev);
573 }
574 return 0;
575 }
576
hidma_resume(struct dma_chan * chan)577 static int hidma_resume(struct dma_chan *chan)
578 {
579 struct hidma_chan *mchan;
580 struct hidma_dev *dmadev;
581 int rc = 0;
582
583 mchan = to_hidma_chan(chan);
584 dmadev = to_hidma_dev(mchan->chan.device);
585 if (mchan->paused) {
586 pm_runtime_get_sync(dmadev->ddev.dev);
587 rc = hidma_ll_enable(dmadev->lldev);
588 if (!rc)
589 mchan->paused = false;
590 else
591 dev_err(dmadev->ddev.dev,
592 "failed to resume the channel");
593 pm_runtime_mark_last_busy(dmadev->ddev.dev);
594 pm_runtime_put_autosuspend(dmadev->ddev.dev);
595 }
596 return rc;
597 }
598
hidma_chirq_handler(int chirq,void * arg)599 static irqreturn_t hidma_chirq_handler(int chirq, void *arg)
600 {
601 struct hidma_lldev *lldev = arg;
602
603 /*
604 * All interrupts are request driven.
605 * HW doesn't send an interrupt by itself.
606 */
607 return hidma_ll_inthandler(chirq, lldev);
608 }
609
610 #ifdef CONFIG_GENERIC_MSI_IRQ
hidma_chirq_handler_msi(int chirq,void * arg)611 static irqreturn_t hidma_chirq_handler_msi(int chirq, void *arg)
612 {
613 struct hidma_lldev **lldevp = arg;
614 struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp);
615
616 return hidma_ll_inthandler_msi(chirq, *lldevp,
617 1 << (chirq - dmadev->msi_virqbase));
618 }
619 #endif
620
hidma_show_values(struct device * dev,struct device_attribute * attr,char * buf)621 static ssize_t hidma_show_values(struct device *dev,
622 struct device_attribute *attr, char *buf)
623 {
624 struct hidma_dev *mdev = dev_get_drvdata(dev);
625
626 if (strcmp(attr->attr.name, "chid") == 0)
627 return sysfs_emit(buf, "%d\n", mdev->chidx);
628
629 return 0;
630 }
631
hidma_sysfs_uninit(struct hidma_dev * dev)632 static inline void hidma_sysfs_uninit(struct hidma_dev *dev)
633 {
634 device_remove_file(dev->ddev.dev, dev->chid_attrs);
635 }
636
637 static struct device_attribute*
hidma_create_sysfs_entry(struct hidma_dev * dev,char * name,int mode)638 hidma_create_sysfs_entry(struct hidma_dev *dev, char *name, int mode)
639 {
640 struct device_attribute *attrs;
641 char *name_copy;
642
643 attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute),
644 GFP_KERNEL);
645 if (!attrs)
646 return NULL;
647
648 name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL);
649 if (!name_copy)
650 return NULL;
651
652 attrs->attr.name = name_copy;
653 attrs->attr.mode = mode;
654 attrs->show = hidma_show_values;
655 sysfs_attr_init(&attrs->attr);
656
657 return attrs;
658 }
659
hidma_sysfs_init(struct hidma_dev * dev)660 static int hidma_sysfs_init(struct hidma_dev *dev)
661 {
662 dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO);
663 if (!dev->chid_attrs)
664 return -ENOMEM;
665
666 return device_create_file(dev->ddev.dev, dev->chid_attrs);
667 }
668
669 #ifdef CONFIG_GENERIC_MSI_IRQ
hidma_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)670 static void hidma_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
671 {
672 struct device *dev = msi_desc_to_dev(desc);
673 struct hidma_dev *dmadev = dev_get_drvdata(dev);
674
675 if (!desc->msi_index) {
676 writel(msg->address_lo, dmadev->dev_evca + 0x118);
677 writel(msg->address_hi, dmadev->dev_evca + 0x11C);
678 writel(msg->data, dmadev->dev_evca + 0x120);
679 }
680 }
681 #endif
682
hidma_free_msis(struct hidma_dev * dmadev)683 static void hidma_free_msis(struct hidma_dev *dmadev)
684 {
685 #ifdef CONFIG_GENERIC_MSI_IRQ
686 struct device *dev = dmadev->ddev.dev;
687 int i, virq;
688
689 for (i = 0; i < HIDMA_MSI_INTS; i++) {
690 virq = msi_get_virq(dev, i);
691 if (virq)
692 devm_free_irq(dev, virq, &dmadev->lldev);
693 }
694
695 platform_device_msi_free_irqs_all(dev);
696 #endif
697 }
698
hidma_request_msi(struct hidma_dev * dmadev,struct platform_device * pdev)699 static int hidma_request_msi(struct hidma_dev *dmadev,
700 struct platform_device *pdev)
701 {
702 #ifdef CONFIG_GENERIC_MSI_IRQ
703 int rc, i, virq;
704
705 rc = platform_device_msi_init_and_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS,
706 hidma_write_msi_msg);
707 if (rc)
708 return rc;
709
710 for (i = 0; i < HIDMA_MSI_INTS; i++) {
711 virq = msi_get_virq(&pdev->dev, i);
712 rc = devm_request_irq(&pdev->dev, virq,
713 hidma_chirq_handler_msi,
714 0, "qcom-hidma-msi",
715 &dmadev->lldev);
716 if (rc)
717 break;
718 if (!i)
719 dmadev->msi_virqbase = virq;
720 }
721
722 if (rc) {
723 /* free allocated MSI interrupts above */
724 for (--i; i >= 0; i--) {
725 virq = msi_get_virq(&pdev->dev, i);
726 devm_free_irq(&pdev->dev, virq, &dmadev->lldev);
727 }
728 dev_warn(&pdev->dev,
729 "failed to request MSI irq, falling back to wired IRQ\n");
730 } else {
731 /* Add callback to free MSIs on teardown */
732 hidma_ll_setup_irq(dmadev->lldev, true);
733 }
734 return rc;
735 #else
736 return -EINVAL;
737 #endif
738 }
739
hidma_test_capability(struct device * dev,enum hidma_cap test_cap)740 static bool hidma_test_capability(struct device *dev, enum hidma_cap test_cap)
741 {
742 enum hidma_cap cap;
743
744 cap = (uintptr_t) device_get_match_data(dev);
745 return cap ? ((cap & test_cap) > 0) : 0;
746 }
747
hidma_probe(struct platform_device * pdev)748 static int hidma_probe(struct platform_device *pdev)
749 {
750 struct hidma_dev *dmadev;
751 struct resource *trca_resource;
752 struct resource *evca_resource;
753 int chirq;
754 void __iomem *evca;
755 void __iomem *trca;
756 int rc;
757 bool msi;
758
759 pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT);
760 pm_runtime_use_autosuspend(&pdev->dev);
761 pm_runtime_set_active(&pdev->dev);
762 pm_runtime_enable(&pdev->dev);
763
764 trca = devm_platform_get_and_ioremap_resource(pdev, 0, &trca_resource);
765 if (IS_ERR(trca)) {
766 rc = PTR_ERR(trca);
767 goto bailout;
768 }
769
770 evca = devm_platform_get_and_ioremap_resource(pdev, 1, &evca_resource);
771 if (IS_ERR(evca)) {
772 rc = PTR_ERR(evca);
773 goto bailout;
774 }
775
776 /*
777 * This driver only handles the channel IRQs.
778 * Common IRQ is handled by the management driver.
779 */
780 chirq = platform_get_irq(pdev, 0);
781 if (chirq < 0) {
782 rc = chirq;
783 goto bailout;
784 }
785
786 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);
787 if (!dmadev) {
788 rc = -ENOMEM;
789 goto bailout;
790 }
791
792 INIT_LIST_HEAD(&dmadev->ddev.channels);
793 spin_lock_init(&dmadev->lock);
794 dmadev->ddev.dev = &pdev->dev;
795 pm_runtime_get_sync(dmadev->ddev.dev);
796
797 dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask);
798 dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask);
799 if (WARN_ON(!pdev->dev.dma_mask)) {
800 rc = -ENXIO;
801 goto dmafree;
802 }
803
804 dmadev->dev_evca = evca;
805 dmadev->evca_resource = evca_resource;
806 dmadev->dev_trca = trca;
807 dmadev->trca_resource = trca_resource;
808 dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy;
809 dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset;
810 dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources;
811 dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources;
812 dmadev->ddev.device_tx_status = hidma_tx_status;
813 dmadev->ddev.device_issue_pending = hidma_issue_pending;
814 dmadev->ddev.device_pause = hidma_pause;
815 dmadev->ddev.device_resume = hidma_resume;
816 dmadev->ddev.device_terminate_all = hidma_terminate_all;
817 dmadev->ddev.copy_align = 8;
818
819 /*
820 * Determine the MSI capability of the platform. Old HW doesn't
821 * support MSI.
822 */
823 msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP);
824 device_property_read_u32(&pdev->dev, "desc-count",
825 &dmadev->nr_descriptors);
826
827 if (nr_desc_prm) {
828 dev_info(&pdev->dev, "overriding number of descriptors as %d\n",
829 nr_desc_prm);
830 dmadev->nr_descriptors = nr_desc_prm;
831 }
832
833 if (!dmadev->nr_descriptors)
834 dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC;
835
836 if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP))
837 dmadev->chidx = readl(dmadev->dev_trca + 0x40);
838 else
839 dmadev->chidx = readl(dmadev->dev_trca + 0x28);
840
841 /* Set DMA mask to 64 bits. */
842 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
843 if (rc) {
844 dev_warn(&pdev->dev, "unable to set coherent mask to 64");
845 goto dmafree;
846 }
847
848 dmadev->lldev = hidma_ll_init(dmadev->ddev.dev,
849 dmadev->nr_descriptors, dmadev->dev_trca,
850 dmadev->dev_evca, dmadev->chidx);
851 if (!dmadev->lldev) {
852 rc = -EPROBE_DEFER;
853 goto dmafree;
854 }
855
856 platform_set_drvdata(pdev, dmadev);
857 if (msi)
858 rc = hidma_request_msi(dmadev, pdev);
859
860 if (!msi || rc) {
861 hidma_ll_setup_irq(dmadev->lldev, false);
862 rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler,
863 0, "qcom-hidma", dmadev->lldev);
864 if (rc)
865 goto uninit;
866 }
867
868 INIT_LIST_HEAD(&dmadev->ddev.channels);
869 rc = hidma_chan_init(dmadev, 0);
870 if (rc)
871 goto uninit;
872
873 rc = dma_async_device_register(&dmadev->ddev);
874 if (rc)
875 goto uninit;
876
877 dmadev->irq = chirq;
878 tasklet_setup(&dmadev->task, hidma_issue_task);
879 hidma_debug_init(dmadev);
880 hidma_sysfs_init(dmadev);
881 dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n");
882 pm_runtime_mark_last_busy(dmadev->ddev.dev);
883 pm_runtime_put_autosuspend(dmadev->ddev.dev);
884 return 0;
885
886 uninit:
887 if (msi)
888 hidma_free_msis(dmadev);
889
890 hidma_ll_uninit(dmadev->lldev);
891 dmafree:
892 if (dmadev)
893 hidma_free(dmadev);
894 bailout:
895 pm_runtime_put_sync(&pdev->dev);
896 pm_runtime_disable(&pdev->dev);
897 return rc;
898 }
899
hidma_shutdown(struct platform_device * pdev)900 static void hidma_shutdown(struct platform_device *pdev)
901 {
902 struct hidma_dev *dmadev = platform_get_drvdata(pdev);
903
904 dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n");
905
906 pm_runtime_get_sync(dmadev->ddev.dev);
907 if (hidma_ll_disable(dmadev->lldev))
908 dev_warn(dmadev->ddev.dev, "channel did not stop\n");
909 pm_runtime_mark_last_busy(dmadev->ddev.dev);
910 pm_runtime_put_autosuspend(dmadev->ddev.dev);
911
912 }
913
hidma_remove(struct platform_device * pdev)914 static void hidma_remove(struct platform_device *pdev)
915 {
916 struct hidma_dev *dmadev = platform_get_drvdata(pdev);
917
918 pm_runtime_get_sync(dmadev->ddev.dev);
919 dma_async_device_unregister(&dmadev->ddev);
920 if (!dmadev->lldev->msi_support)
921 devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);
922 else
923 hidma_free_msis(dmadev);
924
925 tasklet_kill(&dmadev->task);
926 hidma_sysfs_uninit(dmadev);
927 hidma_debug_uninit(dmadev);
928 hidma_ll_uninit(dmadev->lldev);
929 hidma_free(dmadev);
930
931 dev_info(&pdev->dev, "HI-DMA engine removed\n");
932 pm_runtime_put_sync_suspend(&pdev->dev);
933 pm_runtime_disable(&pdev->dev);
934 }
935
936 #if IS_ENABLED(CONFIG_ACPI)
937 static const struct acpi_device_id hidma_acpi_ids[] = {
938 {"QCOM8061"},
939 {"QCOM8062", HIDMA_MSI_CAP},
940 {"QCOM8063", (HIDMA_MSI_CAP | HIDMA_IDENTITY_CAP)},
941 {},
942 };
943 MODULE_DEVICE_TABLE(acpi, hidma_acpi_ids);
944 #endif
945
946 static struct platform_driver hidma_driver = {
947 .probe = hidma_probe,
948 .remove = hidma_remove,
949 .shutdown = hidma_shutdown,
950 .driver = {
951 .name = "hidma",
952 .acpi_match_table = ACPI_PTR(hidma_acpi_ids),
953 },
954 };
955
956 module_platform_driver(hidma_driver);
957 MODULE_DESCRIPTION("Qualcomm Technologies HIDMA Channel support");
958 MODULE_LICENSE("GPL v2");
959