ivpu_mmu.c (a2fd4a6fae7d7542816452da97bbabfd49a0874c) ivpu_mmu.c (b563e47957af4ff71736c5cc4120a59b055ab583)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/highmem.h>
8

--- 129 unchanged lines hidden (view full) ---

138#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)
139#define IVPU_MMU_CD_0_AA64 BIT(41)
140#define IVPU_MMU_CD_0_S BIT(44)
141#define IVPU_MMU_CD_0_R BIT(45)
142#define IVPU_MMU_CD_0_A BIT(46)
143#define IVPU_MMU_CD_0_ASET BIT(47)
144#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)
145
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2020-2023 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/highmem.h>
8

--- 129 unchanged lines hidden (view full) ---

138#define IVPU_MMU_CD_0_TCR_TBI0 BIT_ULL(38)
139#define IVPU_MMU_CD_0_AA64 BIT(41)
140#define IVPU_MMU_CD_0_S BIT(44)
141#define IVPU_MMU_CD_0_R BIT(45)
142#define IVPU_MMU_CD_0_A BIT(46)
143#define IVPU_MMU_CD_0_ASET BIT(47)
144#define IVPU_MMU_CD_0_ASID GENMASK_ULL(63, 48)
145
146#define IVPU_MMU_T0SZ_48BIT 16
147#define IVPU_MMU_T0SZ_38BIT 26
148
149#define IVPU_MMU_IPS_48BIT 5
150#define IVPU_MMU_IPS_44BIT 4
151#define IVPU_MMU_IPS_42BIT 3
152#define IVPU_MMU_IPS_40BIT 2
153#define IVPU_MMU_IPS_36BIT 1
154#define IVPU_MMU_IPS_32BIT 0
155
156#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
157
158#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)
159#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)
160#define IVPU_MMU_STE_0_S1FMT_LINEAR 0
161#define IVPU_MMU_STE_DWORDS 8
162#define IVPU_MMU_STE_0_CFG_S1_TRANS 5
163#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)

--- 428 unchanged lines hidden (view full) ---

592 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
593
594 return 0;
595}
596
597int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
598{
599 struct ivpu_mmu_info *mmu = vdev->mmu;
146#define IVPU_MMU_CD_1_TTB0_MASK GENMASK_ULL(51, 4)
147
148#define IVPU_MMU_STE_0_S1CDMAX GENMASK_ULL(63, 59)
149#define IVPU_MMU_STE_0_S1FMT GENMASK_ULL(5, 4)
150#define IVPU_MMU_STE_0_S1FMT_LINEAR 0
151#define IVPU_MMU_STE_DWORDS 8
152#define IVPU_MMU_STE_0_CFG_S1_TRANS 5
153#define IVPU_MMU_STE_0_CFG GENMASK_ULL(3, 1)

--- 428 unchanged lines hidden (view full) ---

582 ivpu_mmu_strtab_link_cd(vdev, IVPU_MMU_STREAM_ID3);
583
584 return 0;
585}
586
587int ivpu_mmu_invalidate_tlb(struct ivpu_device *vdev, u16 ssid)
588{
589 struct ivpu_mmu_info *mmu = vdev->mmu;
600 int ret;
590 int ret = 0;
601
591
602 ret = mutex_lock_interruptible(&mmu->lock);
603 if (ret)
604 return ret;
605
606 if (!mmu->on) {
607 ret = 0;
592 mutex_lock(&mmu->lock);
593 if (!mmu->on)
608 goto unlock;
594 goto unlock;
609 }
610
611 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
612 if (ret)
613 goto unlock;
614
615 ret = ivpu_mmu_cmdq_sync(vdev);
616unlock:
617 mutex_unlock(&mmu->lock);
618 return ret;
619}
620
621static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)
622{
623 struct ivpu_mmu_info *mmu = vdev->mmu;
624 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
625 u64 *entry;
626 u64 cd[4];
595
596 ret = ivpu_mmu_cmdq_write_tlbi_nh_asid(vdev, ssid);
597 if (ret)
598 goto unlock;
599
600 ret = ivpu_mmu_cmdq_sync(vdev);
601unlock:
602 mutex_unlock(&mmu->lock);
603 return ret;
604}
605
606static int ivpu_mmu_cd_add(struct ivpu_device *vdev, u32 ssid, u64 cd_dma)
607{
608 struct ivpu_mmu_info *mmu = vdev->mmu;
609 struct ivpu_mmu_cdtab *cdtab = &mmu->cdtab;
610 u64 *entry;
611 u64 cd[4];
627 int ret;
612 int ret = 0;
628
629 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
630 return -EINVAL;
631
632 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
633
634 if (cd_dma != 0) {
613
614 if (ssid > IVPU_MMU_CDTAB_ENT_COUNT)
615 return -EINVAL;
616
617 entry = cdtab->base + (ssid * IVPU_MMU_CDTAB_ENT_SIZE);
618
619 if (cd_dma != 0) {
635 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, IVPU_MMU_T0SZ_48BIT) |
620 cd[0] = FIELD_PREP(IVPU_MMU_CD_0_TCR_T0SZ, 26) |
636 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |
637 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |
638 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |
639 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |
621 FIELD_PREP(IVPU_MMU_CD_0_TCR_TG0, 0) |
622 FIELD_PREP(IVPU_MMU_CD_0_TCR_IRGN0, 0) |
623 FIELD_PREP(IVPU_MMU_CD_0_TCR_ORGN0, 0) |
624 FIELD_PREP(IVPU_MMU_CD_0_TCR_SH0, 0) |
640 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, IVPU_MMU_IPS_48BIT) |
625 FIELD_PREP(IVPU_MMU_CD_0_TCR_IPS, 3) |
641 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |
642 IVPU_MMU_CD_0_TCR_EPD1 |
643 IVPU_MMU_CD_0_AA64 |
644 IVPU_MMU_CD_0_R |
645 IVPU_MMU_CD_0_ASET |
646 IVPU_MMU_CD_0_V;
647 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
648 cd[2] = 0;

--- 11 unchanged lines hidden (view full) ---

660 WRITE_ONCE(entry[3], cd[3]);
661 WRITE_ONCE(entry[0], cd[0]);
662
663 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
664
665 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
666 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
667
626 FIELD_PREP(IVPU_MMU_CD_0_ASID, ssid) |
627 IVPU_MMU_CD_0_TCR_EPD1 |
628 IVPU_MMU_CD_0_AA64 |
629 IVPU_MMU_CD_0_R |
630 IVPU_MMU_CD_0_ASET |
631 IVPU_MMU_CD_0_V;
632 cd[1] = cd_dma & IVPU_MMU_CD_1_TTB0_MASK;
633 cd[2] = 0;

--- 11 unchanged lines hidden (view full) ---

645 WRITE_ONCE(entry[3], cd[3]);
646 WRITE_ONCE(entry[0], cd[0]);
647
648 clflush_cache_range(entry, IVPU_MMU_CDTAB_ENT_SIZE);
649
650 ivpu_dbg(vdev, MMU, "CDTAB %s entry (SSID=%u, dma=%pad): 0x%llx, 0x%llx, 0x%llx, 0x%llx\n",
651 cd_dma ? "write" : "clear", ssid, &cd_dma, cd[0], cd[1], cd[2], cd[3]);
652
668 ret = mutex_lock_interruptible(&mmu->lock);
669 if (ret)
670 return ret;
671
672 if (!mmu->on) {
673 ret = 0;
653 mutex_lock(&mmu->lock);
654 if (!mmu->on)
674 goto unlock;
655 goto unlock;
675 }
676
677 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
678 if (ret)
679 goto unlock;
680
681 ret = ivpu_mmu_cmdq_sync(vdev);
682unlock:
683 mutex_unlock(&mmu->lock);

--- 210 unchanged lines hidden --- 		656
657 ret = ivpu_mmu_cmdq_write_cfgi_all(vdev);
658 if (ret)
659 goto unlock;
660
661 ret = ivpu_mmu_cmdq_sync(vdev);
662unlock:
663 mutex_unlock(&mmu->lock);

--- 210 unchanged lines hidden ---