Lines Matching +full:use +full:- +full:dma +full:- +full:tx
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Driver for Audio DMA Controller (ADMAC) on t8103 (M1) and other Apple chips
102 * correctness. Typical use of the driver (per channel) will be
115 * SRAM_CARVEOUT has 16-bit fields, so the SRAM cannot be larger than
116 * 64K and a 32-bit bitfield over 2K blocks covers it.
122 struct dma_device dma; member
137 struct dma_async_tx_descriptor tx; member
156 ad->txcache.size = readl_relaxed(ad->base + REG_TX_SRAM_SIZE); in admac_alloc_sram_carveout()
157 ad->rxcache.size = readl_relaxed(ad->base + REG_RX_SRAM_SIZE); in admac_alloc_sram_carveout()
160 sram = &ad->txcache; in admac_alloc_sram_carveout()
162 sram = &ad->rxcache; in admac_alloc_sram_carveout()
164 mutex_lock(&ad->cache_alloc_lock); in admac_alloc_sram_carveout()
166 nblocks = sram->size / SRAM_BLOCK; in admac_alloc_sram_carveout()
168 if (!(sram->allocated & BIT(i))) in admac_alloc_sram_carveout()
174 sram->allocated |= BIT(i); in admac_alloc_sram_carveout()
176 ret = -EBUSY; in admac_alloc_sram_carveout()
179 mutex_unlock(&ad->cache_alloc_lock); in admac_alloc_sram_carveout()
193 sram = &ad->txcache; in admac_free_sram_carveout()
195 sram = &ad->rxcache; in admac_free_sram_carveout()
197 if (WARN_ON(base >= sram->size)) in admac_free_sram_carveout()
200 mutex_lock(&ad->cache_alloc_lock); in admac_free_sram_carveout()
202 sram->allocated &= ~BIT(i); in admac_free_sram_carveout()
203 mutex_unlock(&ad->cache_alloc_lock); in admac_free_sram_carveout()
208 void __iomem *addr = ad->base + reg; in admac_modify()
219 static struct admac_tx *to_admac_tx(struct dma_async_tx_descriptor *tx) in to_admac_tx() argument
221 return container_of(tx, struct admac_tx, tx); in to_admac_tx()
230 static dma_cookie_t admac_tx_submit(struct dma_async_tx_descriptor *tx) in admac_tx_submit() argument
232 struct admac_tx *adtx = to_admac_tx(tx); in admac_tx_submit()
233 struct admac_chan *adchan = to_admac_chan(tx->chan); in admac_tx_submit()
237 spin_lock_irqsave(&adchan->lock, flags); in admac_tx_submit()
238 cookie = dma_cookie_assign(tx); in admac_tx_submit()
239 list_add_tail(&adtx->node, &adchan->submitted); in admac_tx_submit()
240 spin_unlock_irqrestore(&adchan->lock, flags); in admac_tx_submit()
245 static int admac_desc_free(struct dma_async_tx_descriptor *tx) in admac_desc_free() argument
247 kfree(to_admac_tx(tx)); in admac_desc_free()
260 if (direction != admac_chan_direction(adchan->no)) in admac_prep_dma_cyclic()
267 adtx->cyclic = true; in admac_prep_dma_cyclic()
269 adtx->buf_addr = buf_addr; in admac_prep_dma_cyclic()
270 adtx->buf_len = buf_len; in admac_prep_dma_cyclic()
271 adtx->buf_end = buf_addr + buf_len; in admac_prep_dma_cyclic()
272 adtx->period_len = period_len; in admac_prep_dma_cyclic()
274 adtx->submitted_pos = 0; in admac_prep_dma_cyclic()
275 adtx->reclaimed_pos = 0; in admac_prep_dma_cyclic()
277 dma_async_tx_descriptor_init(&adtx->tx, chan); in admac_prep_dma_cyclic()
278 adtx->tx.tx_submit = admac_tx_submit; in admac_prep_dma_cyclic()
279 adtx->tx.desc_free = admac_desc_free; in admac_prep_dma_cyclic()
281 return &adtx->tx; in admac_prep_dma_cyclic()
288 struct admac_tx *tx) in admac_cyclic_write_one_desc() argument
292 addr = tx->buf_addr + (tx->submitted_pos % tx->buf_len); in admac_cyclic_write_one_desc()
295 WARN_ON_ONCE(addr + tx->period_len > tx->buf_end); in admac_cyclic_write_one_desc()
297 dev_dbg(ad->dev, "ch%d descriptor: addr=0x%pad len=0x%zx flags=0x%lx\n", in admac_cyclic_write_one_desc()
298 channo, &addr, tx->period_len, FLAG_DESC_NOTIFY); in admac_cyclic_write_one_desc()
300 writel_relaxed(lower_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); in admac_cyclic_write_one_desc()
301 writel_relaxed(upper_32_bits(addr), ad->base + REG_DESC_WRITE(channo)); in admac_cyclic_write_one_desc()
302 writel_relaxed(tx->period_len, ad->base + REG_DESC_WRITE(channo)); in admac_cyclic_write_one_desc()
303 writel_relaxed(FLAG_DESC_NOTIFY, ad->base + REG_DESC_WRITE(channo)); in admac_cyclic_write_one_desc()
305 tx->submitted_pos += tx->period_len; in admac_cyclic_write_one_desc()
306 tx->submitted_pos %= 2 * tx->buf_len; in admac_cyclic_write_one_desc()
314 struct admac_tx *tx) in admac_cyclic_write_desc() argument
319 if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_FULL) in admac_cyclic_write_desc()
321 admac_cyclic_write_one_desc(ad, channo, tx); in admac_cyclic_write_desc()
331 return (wrslot + 4 - rdslot) % 4; in admac_ring_noccupied_slots()
353 ring1 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); in admac_cyclic_read_residue()
354 residue1 = readl_relaxed(ad->base + REG_RESIDUE(channo)); in admac_cyclic_read_residue()
355 ring2 = readl_relaxed(ad->base + REG_REPORT_RING(channo)); in admac_cyclic_read_residue()
356 residue2 = readl_relaxed(ad->base + REG_RESIDUE(channo)); in admac_cyclic_read_residue()
365 /* No descriptor load between the two reads, ring2 is safe to use */ in admac_cyclic_read_residue()
369 pos = adtx->reclaimed_pos + adtx->period_len * (nreports + 1) - residue2; in admac_cyclic_read_residue()
371 return adtx->buf_len - pos % adtx->buf_len; in admac_cyclic_read_residue()
378 struct admac_data *ad = adchan->host; in admac_tx_status()
389 spin_lock_irqsave(&adchan->lock, flags); in admac_tx_status()
390 adtx = adchan->current_tx; in admac_tx_status()
392 if (adtx && adtx->tx.cookie == cookie) { in admac_tx_status()
394 residue = admac_cyclic_read_residue(ad, adchan->no, adtx); in admac_tx_status()
398 list_for_each_entry(adtx, &adchan->issued, node) { in admac_tx_status()
399 if (adtx->tx.cookie == cookie) { in admac_tx_status()
400 residue = adtx->buf_len; in admac_tx_status()
405 spin_unlock_irqrestore(&adchan->lock, flags); in admac_tx_status()
413 struct admac_data *ad = adchan->host; in admac_start_chan()
414 u32 startbit = 1 << (adchan->no / 2); in admac_start_chan()
417 ad->base + REG_CHAN_INTSTATUS(adchan->no, ad->irq_index)); in admac_start_chan()
419 ad->base + REG_CHAN_INTMASK(adchan->no, ad->irq_index)); in admac_start_chan()
421 switch (admac_chan_direction(adchan->no)) { in admac_start_chan()
423 writel_relaxed(startbit, ad->base + REG_TX_START); in admac_start_chan()
426 writel_relaxed(startbit, ad->base + REG_RX_START); in admac_start_chan()
431 dev_dbg(adchan->host->dev, "ch%d start\n", adchan->no); in admac_start_chan()
436 struct admac_data *ad = adchan->host; in admac_stop_chan()
437 u32 stopbit = 1 << (adchan->no / 2); in admac_stop_chan()
439 switch (admac_chan_direction(adchan->no)) { in admac_stop_chan()
441 writel_relaxed(stopbit, ad->base + REG_TX_STOP); in admac_stop_chan()
444 writel_relaxed(stopbit, ad->base + REG_RX_STOP); in admac_stop_chan()
449 dev_dbg(adchan->host->dev, "ch%d stop\n", adchan->no); in admac_stop_chan()
454 struct admac_data *ad = adchan->host; in admac_reset_rings()
457 ad->base + REG_CHAN_CTL(adchan->no)); in admac_reset_rings()
458 writel_relaxed(0, ad->base + REG_CHAN_CTL(adchan->no)); in admac_reset_rings()
463 struct admac_data *ad = adchan->host; in admac_start_current_tx()
464 int ch = adchan->no; in admac_start_current_tx()
467 writel_relaxed(0, ad->base + REG_CHAN_CTL(ch)); in admac_start_current_tx()
469 admac_cyclic_write_one_desc(ad, ch, adchan->current_tx); in admac_start_current_tx()
471 admac_cyclic_write_desc(ad, ch, adchan->current_tx); in admac_start_current_tx()
477 struct admac_tx *tx; in admac_issue_pending() local
480 spin_lock_irqsave(&adchan->lock, flags); in admac_issue_pending()
481 list_splice_tail_init(&adchan->submitted, &adchan->issued); in admac_issue_pending()
482 if (!list_empty(&adchan->issued) && !adchan->current_tx) { in admac_issue_pending()
483 tx = list_first_entry(&adchan->issued, struct admac_tx, node); in admac_issue_pending()
484 list_del(&tx->node); in admac_issue_pending()
486 adchan->current_tx = tx; in admac_issue_pending()
487 adchan->nperiod_acks = 0; in admac_issue_pending()
490 spin_unlock_irqrestore(&adchan->lock, flags); in admac_issue_pending()
516 spin_lock_irqsave(&adchan->lock, flags); in admac_terminate_all()
520 if (adchan->current_tx) { in admac_terminate_all()
521 list_add_tail(&adchan->current_tx->node, &adchan->to_free); in admac_terminate_all()
522 adchan->current_tx = NULL; in admac_terminate_all()
528 list_splice_tail_init(&adchan->submitted, &adchan->to_free); in admac_terminate_all()
529 list_splice_tail_init(&adchan->issued, &adchan->to_free); in admac_terminate_all()
530 spin_unlock_irqrestore(&adchan->lock, flags); in admac_terminate_all()
542 spin_lock_irqsave(&adchan->lock, flags); in admac_synchronize()
543 list_splice_tail_init(&adchan->to_free, &head); in admac_synchronize()
544 spin_unlock_irqrestore(&adchan->lock, flags); in admac_synchronize()
546 tasklet_kill(&adchan->tasklet); in admac_synchronize()
549 list_del(&adtx->node); in admac_synchronize()
550 admac_desc_free(&adtx->tx); in admac_synchronize()
557 struct admac_data *ad = adchan->host; in admac_alloc_chan_resources()
560 dma_cookie_init(&adchan->chan); in admac_alloc_chan_resources()
561 ret = admac_alloc_sram_carveout(ad, admac_chan_direction(adchan->no), in admac_alloc_chan_resources()
562 &adchan->carveout); in admac_alloc_chan_resources()
566 writel_relaxed(adchan->carveout, in admac_alloc_chan_resources()
567 ad->base + REG_CHAN_SRAM_CARVEOUT(adchan->no)); in admac_alloc_chan_resources()
577 admac_free_sram_carveout(adchan->host, admac_chan_direction(adchan->no), in admac_free_chan_resources()
578 adchan->carveout); in admac_free_chan_resources()
584 struct admac_data *ad = (struct admac_data *) ofdma->of_dma_data; in admac_dma_of_xlate()
587 if (dma_spec->args_count != 1) in admac_dma_of_xlate()
590 index = dma_spec->args[0]; in admac_dma_of_xlate()
592 if (index >= ad->nchannels) { in admac_dma_of_xlate()
593 dev_err(ad->dev, "channel index %u out of bounds\n", index); in admac_dma_of_xlate()
597 return dma_get_slave_channel(&ad->channels[index].chan); in admac_dma_of_xlate()
607 if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_EMPTY) in admac_drain_reports()
610 countval_lo = readl_relaxed(ad->base + REG_REPORT_READ(channo)); in admac_drain_reports()
611 countval_hi = readl_relaxed(ad->base + REG_REPORT_READ(channo)); in admac_drain_reports()
612 unk1 = readl_relaxed(ad->base + REG_REPORT_READ(channo)); in admac_drain_reports()
613 flags = readl_relaxed(ad->base + REG_REPORT_READ(channo)); in admac_drain_reports()
615 dev_dbg(ad->dev, "ch%d report: countval=0x%llx unk1=0x%x flags=0x%x\n", in admac_drain_reports()
626 if (readl_relaxed(ad->base + REG_DESC_RING(channo)) & RING_ERR) { in admac_handle_status_err()
627 writel_relaxed(RING_ERR, ad->base + REG_DESC_RING(channo)); in admac_handle_status_err()
628 dev_err_ratelimited(ad->dev, "ch%d descriptor ring error\n", channo); in admac_handle_status_err()
632 if (readl_relaxed(ad->base + REG_REPORT_RING(channo)) & RING_ERR) { in admac_handle_status_err()
633 writel_relaxed(RING_ERR, ad->base + REG_REPORT_RING(channo)); in admac_handle_status_err()
634 dev_err_ratelimited(ad->dev, "ch%d report ring error\n", channo); in admac_handle_status_err()
639 dev_err(ad->dev, "ch%d unknown error, masking errors as cause of IRQs\n", channo); in admac_handle_status_err()
640 admac_modify(ad, REG_CHAN_INTMASK(channo, ad->irq_index), in admac_handle_status_err()
647 struct admac_chan *adchan = &ad->channels[channo]; in admac_handle_status_desc_done()
652 ad->base + REG_CHAN_INTSTATUS(channo, ad->irq_index)); in admac_handle_status_desc_done()
654 spin_lock_irqsave(&adchan->lock, flags); in admac_handle_status_desc_done()
657 if (adchan->current_tx) { in admac_handle_status_desc_done()
658 struct admac_tx *tx = adchan->current_tx; in admac_handle_status_desc_done() local
660 adchan->nperiod_acks += nreports; in admac_handle_status_desc_done()
661 tx->reclaimed_pos += nreports * tx->period_len; in admac_handle_status_desc_done()
662 tx->reclaimed_pos %= 2 * tx->buf_len; in admac_handle_status_desc_done()
664 admac_cyclic_write_desc(ad, channo, tx); in admac_handle_status_desc_done()
665 tasklet_schedule(&adchan->tasklet); in admac_handle_status_desc_done()
667 spin_unlock_irqrestore(&adchan->lock, flags); in admac_handle_status_desc_done()
672 u32 cause = readl_relaxed(ad->base + REG_CHAN_INTSTATUS(no, ad->irq_index)); in admac_handle_chan_int()
687 rx_intstate = readl_relaxed(ad->base + REG_RX_INTSTATE(ad->irq_index)); in admac_interrupt()
688 tx_intstate = readl_relaxed(ad->base + REG_TX_INTSTATE(ad->irq_index)); in admac_interrupt()
689 global_intstate = readl_relaxed(ad->base + REG_GLOBAL_INTSTATE(ad->irq_index)); in admac_interrupt()
694 for (i = 0; i < ad->nchannels; i += 2) { in admac_interrupt()
700 for (i = 1; i < ad->nchannels; i += 2) { in admac_interrupt()
707 dev_warn(ad->dev, "clearing unknown global interrupt flag: %x\n", in admac_interrupt()
709 writel_relaxed(~(u32) 0, ad->base + REG_GLOBAL_INTSTATE(ad->irq_index)); in admac_interrupt()
723 spin_lock_irq(&adchan->lock); in admac_chan_tasklet()
724 adtx = adchan->current_tx; in admac_chan_tasklet()
725 nacks = adchan->nperiod_acks; in admac_chan_tasklet()
726 adchan->nperiod_acks = 0; in admac_chan_tasklet()
727 spin_unlock_irq(&adchan->lock); in admac_chan_tasklet()
735 dmaengine_desc_get_callback(&adtx->tx, &cb); in admac_chan_tasklet()
736 while (nacks--) in admac_chan_tasklet()
744 struct admac_data *ad = adchan->host; in admac_device_config()
745 bool is_tx = admac_chan_direction(adchan->no) == DMA_MEM_TO_DEV; in admac_device_config()
747 u32 bus_width = readl_relaxed(ad->base + REG_BUS_WIDTH(adchan->no)) & in admac_device_config()
750 switch (is_tx ? config->dst_addr_width : config->src_addr_width) { in admac_device_config()
764 return -EINVAL; in admac_device_config()
770 * The controller has some means of out-of-band signalling, to the peripheral, in admac_device_config()
774 switch (is_tx ? config->dst_port_window_size : config->src_port_window_size) { in admac_device_config()
784 return -EINVAL; in admac_device_config()
787 writel_relaxed(bus_width, ad->base + REG_BUS_WIDTH(adchan->no)); in admac_device_config()
791 * held in controller's per-channel FIFO. Transfers seem to be triggered in admac_device_config()
798 ad->base + REG_CHAN_FIFOCTL(adchan->no)); in admac_device_config()
805 struct device_node *np = pdev->dev.of_node; in admac_probe()
807 struct dma_device *dma; in admac_probe() local
811 err = of_property_read_u32(np, "dma-channels", &nchannels); in admac_probe()
813 dev_err(&pdev->dev, "missing or invalid dma-channels property\n"); in admac_probe()
814 return -EINVAL; in admac_probe()
817 ad = devm_kzalloc(&pdev->dev, struct_size(ad, channels, nchannels), GFP_KERNEL); in admac_probe()
819 return -ENOMEM; in admac_probe()
822 ad->dev = &pdev->dev; in admac_probe()
823 ad->nchannels = nchannels; in admac_probe()
824 mutex_init(&ad->cache_alloc_lock); in admac_probe()
828 * we find one we can use. in admac_probe()
833 ad->irq_index = i; in admac_probe()
839 return dev_err_probe(&pdev->dev, irq, "no usable interrupt\n"); in admac_probe()
840 ad->irq = irq; in admac_probe()
842 ad->base = devm_platform_ioremap_resource(pdev, 0); in admac_probe()
843 if (IS_ERR(ad->base)) in admac_probe()
844 return dev_err_probe(&pdev->dev, PTR_ERR(ad->base), in admac_probe()
847 ad->rstc = devm_reset_control_get_optional_shared(&pdev->dev, NULL); in admac_probe()
848 if (IS_ERR(ad->rstc)) in admac_probe()
849 return PTR_ERR(ad->rstc); in admac_probe()
851 dma = &ad->dma; in admac_probe()
853 dma_cap_set(DMA_PRIVATE, dma->cap_mask); in admac_probe()
854 dma_cap_set(DMA_CYCLIC, dma->cap_mask); in admac_probe()
856 dma->dev = &pdev->dev; in admac_probe()
857 dma->device_alloc_chan_resources = admac_alloc_chan_resources; in admac_probe()
858 dma->device_free_chan_resources = admac_free_chan_resources; in admac_probe()
859 dma->device_tx_status = admac_tx_status; in admac_probe()
860 dma->device_issue_pending = admac_issue_pending; in admac_probe()
861 dma->device_terminate_all = admac_terminate_all; in admac_probe()
862 dma->device_synchronize = admac_synchronize; in admac_probe()
863 dma->device_prep_dma_cyclic = admac_prep_dma_cyclic; in admac_probe()
864 dma->device_config = admac_device_config; in admac_probe()
865 dma->device_pause = admac_pause; in admac_probe()
866 dma->device_resume = admac_resume; in admac_probe()
868 dma->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM); in admac_probe()
869 dma->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in admac_probe()
870 dma->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | in admac_probe()
873 dma->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | in admac_probe()
877 INIT_LIST_HEAD(&dma->channels); in admac_probe()
879 struct admac_chan *adchan = &ad->channels[i]; in admac_probe()
881 adchan->host = ad; in admac_probe()
882 adchan->no = i; in admac_probe()
883 adchan->chan.device = &ad->dma; in admac_probe()
884 spin_lock_init(&adchan->lock); in admac_probe()
885 INIT_LIST_HEAD(&adchan->submitted); in admac_probe()
886 INIT_LIST_HEAD(&adchan->issued); in admac_probe()
887 INIT_LIST_HEAD(&adchan->to_free); in admac_probe()
888 list_add_tail(&adchan->chan.device_node, &dma->channels); in admac_probe()
889 tasklet_setup(&adchan->tasklet, admac_chan_tasklet); in admac_probe()
892 err = reset_control_reset(ad->rstc); in admac_probe()
894 return dev_err_probe(&pdev->dev, err, in admac_probe()
897 err = request_irq(irq, admac_interrupt, 0, dev_name(&pdev->dev), ad); in admac_probe()
899 dev_err_probe(&pdev->dev, err, in admac_probe()
904 err = dma_async_device_register(&ad->dma); in admac_probe()
906 dev_err_probe(&pdev->dev, err, "failed to register DMA device\n"); in admac_probe()
910 err = of_dma_controller_register(pdev->dev.of_node, admac_dma_of_xlate, ad); in admac_probe()
912 dma_async_device_unregister(&ad->dma); in admac_probe()
913 dev_err_probe(&pdev->dev, err, "failed to register with OF\n"); in admac_probe()
917 dev_info(&pdev->dev, "Audio DMA Controller\n"); in admac_probe()
922 free_irq(ad->irq, ad); in admac_probe()
924 reset_control_rearm(ad->rstc); in admac_probe()
932 of_dma_controller_free(pdev->dev.of_node); in admac_remove()
933 dma_async_device_unregister(&ad->dma); in admac_remove()
934 free_irq(ad->irq, ad); in admac_remove()
935 reset_control_rearm(ad->rstc); in admac_remove()
946 .name = "apple-admac",
955 MODULE_DESCRIPTION("Driver for Audio DMA Controller (ADMAC) on Apple SoCs");