1 // SPDX-License-Identifier: GPL-2.0+ 2 3 #include <linux/align.h> 4 #include <linux/errno.h> 5 #include <linux/io.h> 6 #include <linux/kernel.h> 7 #include <linux/types.h> 8 9 #include "fdma_pci.h" 10 11 /* When the switch operates as a PCIe endpoint, the FDMA engine needs to 12 * DMA to/from host memory. The FDMA writes to addresses within the endpoint's 13 * internal Outbound (OB) address space, and the PCIe ATU translates these to 14 * DMA addresses on the PCIe bus, targeting host memory. 15 * 16 * The ATU supports up to six outbound regions. This implementation divides 17 * the OB address space into six equally sized chunks. 18 * 19 * +-------------+------------+------------+-----+------------+ 20 * | Index | Region 0 | Region 1 | ... | Region 5 | 21 * +-------------+------------+------------+-----+------------+ 22 * | Base addr | 0x10000000 | 0x12aa0000 | ... | 0x1d520000 | 23 * | Limit addr | 0x12a9ffff | 0x1553ffff | ... | 0x1ffbffff | 24 * | Target addr | host dma | host dma | ... | host dma | 25 * +-------------+------------+------------+-----+------------+ 26 * 27 * Base addr is the start address of the region within the OB address space. 28 * Limit addr is each region's own upper bound. The value actually 29 * programmed is set per-mapping as base_addr + mapped size - 1, and is 30 * usually smaller than this. 31 * Target addr is the host DMA address that the base addr translates to. 32 */ 33 34 #define FDMA_PCI_ATU_OB_START 0x10000000 35 #define FDMA_PCI_ATU_OB_END 0x1fffffff 36 37 #define FDMA_PCI_ATU_ADDR 0x300000 38 #define FDMA_PCI_ATU_IDX_SIZE 0x200 39 #define FDMA_PCI_ATU_ENA_REG 0x4 40 #define FDMA_PCI_ATU_ENA_BIT BIT(31) 41 #define FDMA_PCI_ATU_LWR_BASE_ADDR 0x8 42 #define FDMA_PCI_ATU_UPP_BASE_ADDR 0xc 43 #define FDMA_PCI_ATU_LIMIT_ADDR 0x10 44 #define FDMA_PCI_ATU_LWR_TARGET_ADDR 0x14 45 #define FDMA_PCI_ATU_UPP_TARGET_ADDR 0x18 46 47 static u32 fdma_pci_atu_region_size(void) 48 { 49 return round_down((FDMA_PCI_ATU_OB_END - FDMA_PCI_ATU_OB_START) / 50 FDMA_PCI_ATU_REGION_MAX, FDMA_PCI_ATU_REGION_ALIGN); 51 } 52 53 static void __iomem *fdma_pci_atu_addr_get(void __iomem *addr, int offset, 54 int idx) 55 { 56 return addr + FDMA_PCI_ATU_ADDR + FDMA_PCI_ATU_IDX_SIZE * idx + offset; 57 } 58 59 static void fdma_pci_atu_region_enable(struct fdma_pci_atu_region *region) 60 { 61 writel(FDMA_PCI_ATU_ENA_BIT, 62 fdma_pci_atu_addr_get(region->atu->addr, FDMA_PCI_ATU_ENA_REG, 63 region->idx)); 64 } 65 66 static void fdma_pci_atu_region_disable(struct fdma_pci_atu_region *region) 67 { 68 writel(0, fdma_pci_atu_addr_get(region->atu->addr, FDMA_PCI_ATU_ENA_REG, 69 region->idx)); 70 } 71 72 /* Configure the address translation in the ATU. */ 73 static void 74 fdma_pci_atu_configure_translation(struct fdma_pci_atu_region *region) 75 { 76 struct fdma_pci_atu *atu = region->atu; 77 int idx = region->idx; 78 79 writel(lower_32_bits(region->base_addr), 80 fdma_pci_atu_addr_get(atu->addr, 81 FDMA_PCI_ATU_LWR_BASE_ADDR, idx)); 82 83 writel(upper_32_bits(region->base_addr), 84 fdma_pci_atu_addr_get(atu->addr, 85 FDMA_PCI_ATU_UPP_BASE_ADDR, idx)); 86 87 /* The OB address space lies entirely below 4GB, so the limit always 88 * fits the lower limit register and the upper one is left alone. 89 */ 90 writel(region->limit_addr, 91 fdma_pci_atu_addr_get(atu->addr, FDMA_PCI_ATU_LIMIT_ADDR, idx)); 92 93 writel(lower_32_bits(region->target_addr), 94 fdma_pci_atu_addr_get(atu->addr, 95 FDMA_PCI_ATU_LWR_TARGET_ADDR, idx)); 96 97 writel(upper_32_bits(region->target_addr), 98 fdma_pci_atu_addr_get(atu->addr, 99 FDMA_PCI_ATU_UPP_TARGET_ADDR, idx)); 100 } 101 102 /* Find an unused ATU region. */ 103 static struct fdma_pci_atu_region * 104 fdma_pci_atu_region_get_free(struct fdma_pci_atu *atu) 105 { 106 struct fdma_pci_atu_region *regions = atu->regions; 107 108 for (int i = 0; i < FDMA_PCI_ATU_REGION_MAX; i++) { 109 if (regions[i].in_use) 110 continue; 111 112 return ®ions[i]; 113 } 114 115 return ERR_PTR(-ENOSPC); 116 } 117 118 /* Unmap an ATU region, clearing its translation and disabling it. */ 119 void fdma_pci_atu_region_unmap(struct fdma_pci_atu_region *region) 120 { 121 if (IS_ERR_OR_NULL(region)) 122 return; 123 124 mutex_lock(®ion->atu->lock); 125 126 region->target_addr = 0; 127 region->in_use = false; 128 129 fdma_pci_atu_region_disable(region); 130 fdma_pci_atu_configure_translation(region); 131 132 mutex_unlock(®ion->atu->lock); 133 } 134 EXPORT_SYMBOL_GPL(fdma_pci_atu_region_unmap); 135 136 /* Map a host DMA address into a free outbound region. */ 137 struct fdma_pci_atu_region * 138 fdma_pci_atu_region_map(struct fdma_pci_atu *atu, u64 target_addr, int size) 139 { 140 struct fdma_pci_atu_region *region; 141 142 if (!atu) 143 return ERR_PTR(-EINVAL); 144 145 if (size <= 0) 146 return ERR_PTR(-EINVAL); 147 148 if (size > fdma_pci_atu_region_size()) 149 return ERR_PTR(-ERANGE); 150 151 /* The ATU region base is only ever aligned to FDMA_PCI_ATU_REGION_ALIGN; 152 * require the same alignment of the host target address, since the ATU 153 * translates addr - target_addr + base_addr and any misalignment here 154 * would shift every translated address by the same amount. 155 */ 156 if (!IS_ALIGNED(target_addr, FDMA_PCI_ATU_REGION_ALIGN)) 157 return ERR_PTR(-EINVAL); 158 159 mutex_lock(&atu->lock); 160 161 region = fdma_pci_atu_region_get_free(atu); 162 if (IS_ERR(region)) { 163 mutex_unlock(&atu->lock); 164 return region; 165 } 166 167 region->target_addr = target_addr; 168 region->limit_addr = region->base_addr + size - 1; 169 region->in_use = true; 170 171 fdma_pci_atu_configure_translation(region); 172 fdma_pci_atu_region_enable(region); 173 174 mutex_unlock(&atu->lock); 175 176 return region; 177 } 178 EXPORT_SYMBOL_GPL(fdma_pci_atu_region_map); 179 180 /* Translate a host DMA address to the corresponding OB address. */ 181 u64 fdma_pci_atu_translate_addr(struct fdma_pci_atu_region *region, u64 addr) 182 { 183 return region->base_addr + (addr - region->target_addr); 184 } 185 EXPORT_SYMBOL_GPL(fdma_pci_atu_translate_addr); 186 187 /* Initialize ATU, dividing the OB space into equally sized regions. */ 188 void fdma_pci_atu_init(struct fdma_pci_atu *atu, void __iomem *addr) 189 { 190 struct fdma_pci_atu_region *regions = atu->regions; 191 u32 region_size = fdma_pci_atu_region_size(); 192 193 atu->addr = addr; 194 mutex_init(&atu->lock); 195 196 for (int i = 0; i < FDMA_PCI_ATU_REGION_MAX; i++) { 197 regions[i].base_addr = 198 FDMA_PCI_ATU_OB_START + (i * region_size); 199 regions[i].limit_addr = 0; 200 regions[i].target_addr = 0; 201 regions[i].idx = i; 202 regions[i].atu = atu; 203 regions[i].in_use = false; 204 205 fdma_pci_atu_region_disable(®ions[i]); 206 } 207 } 208 EXPORT_SYMBOL_GPL(fdma_pci_atu_init); 209