Lines Matching +full:recv +full:- +full:not +full:- +full:empty
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (c) 2016-2023, NVIDIA CORPORATION. All rights reserved.
17 #include <dt-bindings/mailbox/tegra186-hsp.h>
79 void (*recv)(struct tegra_hsp_channel *channel); member
128 return readl(hsp->regs + offset); in tegra_hsp_readl()
134 writel(value, hsp->regs + offset); in tegra_hsp_writel()
140 return readl(channel->regs + offset); in tegra_hsp_channel_readl()
146 writel(value, channel->regs + offset); in tegra_hsp_channel_writel()
153 value = tegra_hsp_channel_readl(&db->channel, HSP_DB_ENABLE); in tegra_hsp_doorbell_can_ring()
163 list_for_each_entry(entry, &hsp->doorbells, list) in __tegra_hsp_doorbell_get()
164 if (entry->master == master) in __tegra_hsp_doorbell_get()
176 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_doorbell_get()
178 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_doorbell_get()
193 value = tegra_hsp_channel_readl(&db->channel, HSP_DB_PENDING); in tegra_hsp_doorbell_irq()
194 tegra_hsp_channel_writel(&db->channel, value, HSP_DB_PENDING); in tegra_hsp_doorbell_irq()
196 spin_lock(&hsp->lock); in tegra_hsp_doorbell_irq()
198 for_each_set_bit(master, &value, hsp->mbox_db.num_chans) { in tegra_hsp_doorbell_irq()
207 * In that case, db->channel.chan will still be NULL here and in tegra_hsp_doorbell_irq()
208 * cause a crash if not properly guarded. in tegra_hsp_doorbell_irq()
213 if (db && db->channel.chan) in tegra_hsp_doorbell_irq()
214 mbox_chan_received_data(db->channel.chan, NULL); in tegra_hsp_doorbell_irq()
217 spin_unlock(&hsp->lock); in tegra_hsp_doorbell_irq()
228 status = tegra_hsp_readl(hsp, HSP_INT_IR) & hsp->mask; in tegra_hsp_shared_irq()
230 /* process EMPTY interrupts first */ in tegra_hsp_shared_irq()
233 for_each_set_bit(bit, &mask, hsp->num_sm) { in tegra_hsp_shared_irq()
234 struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; in tegra_hsp_shared_irq()
236 if (mb->producer) { in tegra_hsp_shared_irq()
238 * Disable EMPTY interrupts until data is sent with in tegra_hsp_shared_irq()
239 * the next message. These interrupts are level- in tegra_hsp_shared_irq()
244 spin_lock(&hsp->lock); in tegra_hsp_shared_irq()
246 hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); in tegra_hsp_shared_irq()
247 tegra_hsp_writel(hsp, hsp->mask, in tegra_hsp_shared_irq()
248 HSP_INT_IE(hsp->shared_irq)); in tegra_hsp_shared_irq()
250 spin_unlock(&hsp->lock); in tegra_hsp_shared_irq()
252 mbox_chan_txdone(mb->channel.chan, 0); in tegra_hsp_shared_irq()
259 for_each_set_bit(bit, &mask, hsp->num_sm) { in tegra_hsp_shared_irq()
260 struct tegra_hsp_mailbox *mb = &hsp->mailboxes[bit]; in tegra_hsp_shared_irq()
262 if (!mb->producer) in tegra_hsp_shared_irq()
263 mb->ops->recv(&mb->channel); in tegra_hsp_shared_irq()
277 db = devm_kzalloc(hsp->dev, sizeof(*db), GFP_KERNEL); in tegra_hsp_doorbell_create()
279 return ERR_PTR(-ENOMEM); in tegra_hsp_doorbell_create()
281 offset = (1 + (hsp->num_sm / 2) + hsp->num_ss + hsp->num_as) * SZ_64K; in tegra_hsp_doorbell_create()
282 offset += index * hsp->soc->reg_stride; in tegra_hsp_doorbell_create()
284 db->channel.regs = hsp->regs + offset; in tegra_hsp_doorbell_create()
285 db->channel.hsp = hsp; in tegra_hsp_doorbell_create()
287 db->name = devm_kstrdup_const(hsp->dev, name, GFP_KERNEL); in tegra_hsp_doorbell_create()
288 db->master = master; in tegra_hsp_doorbell_create()
289 db->index = index; in tegra_hsp_doorbell_create()
291 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_doorbell_create()
292 list_add_tail(&db->list, &hsp->doorbells); in tegra_hsp_doorbell_create()
293 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_doorbell_create()
295 return &db->channel; in tegra_hsp_doorbell_create()
300 struct tegra_hsp_doorbell *db = chan->con_priv; in tegra_hsp_doorbell_send_data()
302 tegra_hsp_channel_writel(&db->channel, 1, HSP_DB_TRIGGER); in tegra_hsp_doorbell_send_data()
309 struct tegra_hsp_doorbell *db = chan->con_priv; in tegra_hsp_doorbell_startup()
310 struct tegra_hsp *hsp = db->channel.hsp; in tegra_hsp_doorbell_startup()
315 if (db->master >= chan->mbox->num_chans) { in tegra_hsp_doorbell_startup()
316 dev_err(chan->mbox->dev, in tegra_hsp_doorbell_startup()
318 db->master); in tegra_hsp_doorbell_startup()
319 return -EINVAL; in tegra_hsp_doorbell_startup()
324 return -ENODEV; in tegra_hsp_doorbell_startup()
332 return -ENODEV; in tegra_hsp_doorbell_startup()
334 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_doorbell_startup()
336 value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE); in tegra_hsp_doorbell_startup()
337 value |= BIT(db->master); in tegra_hsp_doorbell_startup()
338 tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE); in tegra_hsp_doorbell_startup()
340 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_doorbell_startup()
347 struct tegra_hsp_doorbell *db = chan->con_priv; in tegra_hsp_doorbell_shutdown()
348 struct tegra_hsp *hsp = db->channel.hsp; in tegra_hsp_doorbell_shutdown()
357 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_doorbell_shutdown()
359 value = tegra_hsp_channel_readl(&ccplex->channel, HSP_DB_ENABLE); in tegra_hsp_doorbell_shutdown()
360 value &= ~BIT(db->master); in tegra_hsp_doorbell_shutdown()
361 tegra_hsp_channel_writel(&ccplex->channel, value, HSP_DB_ENABLE); in tegra_hsp_doorbell_shutdown()
363 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_doorbell_shutdown()
391 mbox_chan_received_data(channel->chan, msg); in tegra_hsp_sm_recv32()
405 .recv = tegra_hsp_sm_recv32,
436 mbox_chan_received_data(channel->chan, msg); in tegra_hsp_sm_recv128()
450 .recv = tegra_hsp_sm_recv128,
455 struct tegra_hsp_mailbox *mb = chan->con_priv; in tegra_hsp_mailbox_send_data()
456 struct tegra_hsp *hsp = mb->channel.hsp; in tegra_hsp_mailbox_send_data()
459 if (WARN_ON(!mb->producer)) in tegra_hsp_mailbox_send_data()
460 return -EPERM; in tegra_hsp_mailbox_send_data()
462 mb->ops->send(&mb->channel, data); in tegra_hsp_mailbox_send_data()
464 /* enable EMPTY interrupt for the shared mailbox */ in tegra_hsp_mailbox_send_data()
465 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_mailbox_send_data()
467 hsp->mask |= BIT(HSP_INT_EMPTY_SHIFT + mb->index); in tegra_hsp_mailbox_send_data()
468 tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq)); in tegra_hsp_mailbox_send_data()
470 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_mailbox_send_data()
478 struct tegra_hsp_mailbox *mb = chan->con_priv; in tegra_hsp_mailbox_flush()
479 struct tegra_hsp_channel *ch = &mb->channel; in tegra_hsp_mailbox_flush()
489 /* Wait until channel is empty */ in tegra_hsp_mailbox_flush()
490 if (chan->active_req != NULL) in tegra_hsp_mailbox_flush()
499 return -ETIME; in tegra_hsp_mailbox_flush()
504 struct tegra_hsp_mailbox *mb = chan->con_priv; in tegra_hsp_mailbox_startup()
505 struct tegra_hsp_channel *ch = &mb->channel; in tegra_hsp_mailbox_startup()
506 struct tegra_hsp *hsp = mb->channel.hsp; in tegra_hsp_mailbox_startup()
509 chan->txdone_method = TXDONE_BY_IRQ; in tegra_hsp_mailbox_startup()
512 * Shared mailboxes start out as consumers by default. FULL and EMPTY in tegra_hsp_mailbox_startup()
515 * Keep EMPTY interrupts disabled at startup and only enable them when in tegra_hsp_mailbox_startup()
517 * EMPTY interrupts are level-triggered, so keeping EMPTY interrupts in tegra_hsp_mailbox_startup()
522 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_mailbox_startup()
524 if (mb->producer) in tegra_hsp_mailbox_startup()
525 hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); in tegra_hsp_mailbox_startup()
527 hsp->mask |= BIT(HSP_INT_FULL_SHIFT + mb->index); in tegra_hsp_mailbox_startup()
529 tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq)); in tegra_hsp_mailbox_startup()
531 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_mailbox_startup()
533 if (hsp->soc->has_per_mb_ie) { in tegra_hsp_mailbox_startup()
534 if (mb->producer) in tegra_hsp_mailbox_startup()
547 struct tegra_hsp_mailbox *mb = chan->con_priv; in tegra_hsp_mailbox_shutdown()
548 struct tegra_hsp_channel *ch = &mb->channel; in tegra_hsp_mailbox_shutdown()
549 struct tegra_hsp *hsp = mb->channel.hsp; in tegra_hsp_mailbox_shutdown()
552 if (hsp->soc->has_per_mb_ie) { in tegra_hsp_mailbox_shutdown()
553 if (mb->producer) in tegra_hsp_mailbox_shutdown()
561 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_mailbox_shutdown()
563 if (mb->producer) in tegra_hsp_mailbox_shutdown()
564 hsp->mask &= ~BIT(HSP_INT_EMPTY_SHIFT + mb->index); in tegra_hsp_mailbox_shutdown()
566 hsp->mask &= ~BIT(HSP_INT_FULL_SHIFT + mb->index); in tegra_hsp_mailbox_shutdown()
568 tegra_hsp_writel(hsp, hsp->mask, HSP_INT_IE(hsp->shared_irq)); in tegra_hsp_mailbox_shutdown()
570 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_mailbox_shutdown()
584 unsigned int type = args->args[0], master = args->args[1]; in tegra_hsp_db_xlate()
585 struct tegra_hsp_channel *channel = ERR_PTR(-ENODEV); in tegra_hsp_db_xlate()
591 if (type != TEGRA_HSP_MBOX_TYPE_DB || !hsp->doorbell_irq) in tegra_hsp_db_xlate()
592 return ERR_PTR(-ENODEV); in tegra_hsp_db_xlate()
596 channel = &db->channel; in tegra_hsp_db_xlate()
601 spin_lock_irqsave(&hsp->lock, flags); in tegra_hsp_db_xlate()
603 for (i = 0; i < mbox->num_chans; i++) { in tegra_hsp_db_xlate()
604 chan = &mbox->chans[i]; in tegra_hsp_db_xlate()
605 if (!chan->con_priv) { in tegra_hsp_db_xlate()
606 channel->chan = chan; in tegra_hsp_db_xlate()
607 chan->con_priv = db; in tegra_hsp_db_xlate()
614 spin_unlock_irqrestore(&hsp->lock, flags); in tegra_hsp_db_xlate()
616 return chan ?: ERR_PTR(-EBUSY); in tegra_hsp_db_xlate()
623 unsigned int type = args->args[0], index; in tegra_hsp_sm_xlate()
626 index = args->args[1] & TEGRA_HSP_SM_MASK; in tegra_hsp_sm_xlate()
629 !hsp->shared_irqs || index >= hsp->num_sm) in tegra_hsp_sm_xlate()
630 return ERR_PTR(-ENODEV); in tegra_hsp_sm_xlate()
632 mb = &hsp->mailboxes[index]; in tegra_hsp_sm_xlate()
635 if (!hsp->soc->has_128_bit_mb) in tegra_hsp_sm_xlate()
636 return ERR_PTR(-ENODEV); in tegra_hsp_sm_xlate()
638 mb->ops = &tegra_hsp_sm_128bit_ops; in tegra_hsp_sm_xlate()
640 mb->ops = &tegra_hsp_sm_32bit_ops; in tegra_hsp_sm_xlate()
643 if ((args->args[1] & TEGRA_HSP_SM_FLAG_TX) == 0) in tegra_hsp_sm_xlate()
644 mb->producer = false; in tegra_hsp_sm_xlate()
646 mb->producer = true; in tegra_hsp_sm_xlate()
648 return mb->channel.chan; in tegra_hsp_sm_xlate()
653 const struct tegra_hsp_db_map *map = hsp->soc->map; in tegra_hsp_add_doorbells()
656 while (map->name) { in tegra_hsp_add_doorbells()
657 channel = tegra_hsp_doorbell_create(hsp, map->name, in tegra_hsp_add_doorbells()
658 map->master, map->index); in tegra_hsp_add_doorbells()
672 hsp->mailboxes = devm_kcalloc(dev, hsp->num_sm, sizeof(*hsp->mailboxes), in tegra_hsp_add_mailboxes()
674 if (!hsp->mailboxes) in tegra_hsp_add_mailboxes()
675 return -ENOMEM; in tegra_hsp_add_mailboxes()
677 for (i = 0; i < hsp->num_sm; i++) { in tegra_hsp_add_mailboxes()
678 struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i]; in tegra_hsp_add_mailboxes()
680 mb->index = i; in tegra_hsp_add_mailboxes()
682 mb->channel.hsp = hsp; in tegra_hsp_add_mailboxes()
683 mb->channel.regs = hsp->regs + SZ_64K + i * SZ_32K; in tegra_hsp_add_mailboxes()
684 mb->channel.chan = &hsp->mbox_sm.chans[i]; in tegra_hsp_add_mailboxes()
685 mb->channel.chan->con_priv = mb; in tegra_hsp_add_mailboxes()
696 for (i = 0; i < hsp->num_si; i++) { in tegra_hsp_request_shared_irq()
697 irq = hsp->shared_irqs[i]; in tegra_hsp_request_shared_irq()
701 err = devm_request_irq(hsp->dev, irq, tegra_hsp_shared_irq, 0, in tegra_hsp_request_shared_irq()
702 dev_name(hsp->dev), hsp); in tegra_hsp_request_shared_irq()
704 dev_err(hsp->dev, "failed to request interrupt: %d\n", in tegra_hsp_request_shared_irq()
709 hsp->shared_irq = i; in tegra_hsp_request_shared_irq()
712 tegra_hsp_writel(hsp, 0, HSP_INT_IE(hsp->shared_irq)); in tegra_hsp_request_shared_irq()
714 dev_dbg(hsp->dev, "interrupt requested: %u\n", irq); in tegra_hsp_request_shared_irq()
719 if (i == hsp->num_si) { in tegra_hsp_request_shared_irq()
720 dev_err(hsp->dev, "failed to find available interrupt\n"); in tegra_hsp_request_shared_irq()
721 return -ENOENT; in tegra_hsp_request_shared_irq()
734 hsp = devm_kzalloc(&pdev->dev, sizeof(*hsp), GFP_KERNEL); in tegra_hsp_probe()
736 return -ENOMEM; in tegra_hsp_probe()
738 hsp->dev = &pdev->dev; in tegra_hsp_probe()
739 hsp->soc = of_device_get_match_data(&pdev->dev); in tegra_hsp_probe()
740 INIT_LIST_HEAD(&hsp->doorbells); in tegra_hsp_probe()
741 spin_lock_init(&hsp->lock); in tegra_hsp_probe()
743 hsp->regs = devm_platform_ioremap_resource(pdev, 0); in tegra_hsp_probe()
744 if (IS_ERR(hsp->regs)) in tegra_hsp_probe()
745 return PTR_ERR(hsp->regs); in tegra_hsp_probe()
748 hsp->num_sm = (value >> HSP_nSM_SHIFT) & HSP_nINT_MASK; in tegra_hsp_probe()
749 hsp->num_ss = (value >> HSP_nSS_SHIFT) & HSP_nINT_MASK; in tegra_hsp_probe()
750 hsp->num_as = (value >> HSP_nAS_SHIFT) & HSP_nINT_MASK; in tegra_hsp_probe()
751 hsp->num_db = (value >> HSP_nDB_SHIFT) & HSP_nINT_MASK; in tegra_hsp_probe()
752 hsp->num_si = (value >> HSP_nSI_SHIFT) & HSP_nINT_MASK; in tegra_hsp_probe()
756 hsp->doorbell_irq = err; in tegra_hsp_probe()
758 if (hsp->num_si > 0) { in tegra_hsp_probe()
761 hsp->shared_irqs = devm_kcalloc(&pdev->dev, hsp->num_si, in tegra_hsp_probe()
762 sizeof(*hsp->shared_irqs), in tegra_hsp_probe()
764 if (!hsp->shared_irqs) in tegra_hsp_probe()
765 return -ENOMEM; in tegra_hsp_probe()
767 for (i = 0; i < hsp->num_si; i++) { in tegra_hsp_probe()
772 return -ENOMEM; in tegra_hsp_probe()
776 hsp->shared_irqs[i] = err; in tegra_hsp_probe()
784 devm_kfree(&pdev->dev, hsp->shared_irqs); in tegra_hsp_probe()
785 hsp->shared_irqs = NULL; in tegra_hsp_probe()
790 hsp->mbox_db.of_xlate = tegra_hsp_db_xlate; in tegra_hsp_probe()
791 hsp->mbox_db.num_chans = 32; in tegra_hsp_probe()
792 hsp->mbox_db.dev = &pdev->dev; in tegra_hsp_probe()
793 hsp->mbox_db.ops = &tegra_hsp_db_ops; in tegra_hsp_probe()
795 hsp->mbox_db.chans = devm_kcalloc(&pdev->dev, hsp->mbox_db.num_chans, in tegra_hsp_probe()
796 sizeof(*hsp->mbox_db.chans), in tegra_hsp_probe()
798 if (!hsp->mbox_db.chans) in tegra_hsp_probe()
799 return -ENOMEM; in tegra_hsp_probe()
801 if (hsp->doorbell_irq) { in tegra_hsp_probe()
804 dev_err(&pdev->dev, "failed to add doorbells: %d\n", in tegra_hsp_probe()
810 err = devm_mbox_controller_register(&pdev->dev, &hsp->mbox_db); in tegra_hsp_probe()
812 dev_err(&pdev->dev, "failed to register doorbell mailbox: %d\n", in tegra_hsp_probe()
818 hsp->mbox_sm.of_xlate = tegra_hsp_sm_xlate; in tegra_hsp_probe()
819 hsp->mbox_sm.num_chans = hsp->num_sm; in tegra_hsp_probe()
820 hsp->mbox_sm.dev = &pdev->dev; in tegra_hsp_probe()
821 hsp->mbox_sm.ops = &tegra_hsp_sm_ops; in tegra_hsp_probe()
823 hsp->mbox_sm.chans = devm_kcalloc(&pdev->dev, hsp->mbox_sm.num_chans, in tegra_hsp_probe()
824 sizeof(*hsp->mbox_sm.chans), in tegra_hsp_probe()
826 if (!hsp->mbox_sm.chans) in tegra_hsp_probe()
827 return -ENOMEM; in tegra_hsp_probe()
829 if (hsp->shared_irqs) { in tegra_hsp_probe()
830 err = tegra_hsp_add_mailboxes(hsp, &pdev->dev); in tegra_hsp_probe()
832 dev_err(&pdev->dev, "failed to add mailboxes: %d\n", in tegra_hsp_probe()
838 err = devm_mbox_controller_register(&pdev->dev, &hsp->mbox_sm); in tegra_hsp_probe()
840 dev_err(&pdev->dev, "failed to register shared mailbox: %d\n", in tegra_hsp_probe()
847 if (hsp->doorbell_irq) { in tegra_hsp_probe()
848 err = devm_request_irq(&pdev->dev, hsp->doorbell_irq, in tegra_hsp_probe()
850 dev_name(&pdev->dev), hsp); in tegra_hsp_probe()
852 dev_err(&pdev->dev, in tegra_hsp_probe()
854 hsp->doorbell_irq, err); in tegra_hsp_probe()
859 if (hsp->shared_irqs) { in tegra_hsp_probe()
865 lockdep_register_key(&hsp->lock_key); in tegra_hsp_probe()
866 lockdep_set_class(&hsp->lock, &hsp->lock_key); in tegra_hsp_probe()
875 lockdep_unregister_key(&hsp->lock_key); in tegra_hsp_remove()
884 list_for_each_entry(db, &hsp->doorbells, list) { in tegra_hsp_resume()
885 if (db->channel.chan) in tegra_hsp_resume()
886 tegra_hsp_doorbell_startup(db->channel.chan); in tegra_hsp_resume()
889 if (hsp->mailboxes) { in tegra_hsp_resume()
890 for (i = 0; i < hsp->num_sm; i++) { in tegra_hsp_resume()
891 struct tegra_hsp_mailbox *mb = &hsp->mailboxes[i]; in tegra_hsp_resume()
893 if (mb->channel.chan->cl) in tegra_hsp_resume()
894 tegra_hsp_mailbox_startup(mb->channel.chan); in tegra_hsp_resume()
940 { .compatible = "nvidia,tegra186-hsp", .data = &tegra186_hsp_soc },
941 { .compatible = "nvidia,tegra194-hsp", .data = &tegra194_hsp_soc },
942 { .compatible = "nvidia,tegra234-hsp", .data = &tegra234_hsp_soc },
943 { .compatible = "nvidia,tegra264-hsp", .data = &tegra264_hsp_soc },
949 .name = "tegra-hsp",