1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (C) 2025, Advanced Micro Devices, Inc.
4 */
5
6 #include <drm/amdxdna_accel.h>
7 #include <drm/drm_managed.h>
8 #include <linux/iommu.h>
9 #include <linux/iova.h>
10
11 #include "amdxdna_gem.h"
12 #include "amdxdna_pci_drv.h"
13
14 static bool force_iova;
15 module_param(force_iova, bool, 0600);
16 MODULE_PARM_DESC(force_iova, "Force use IOVA (Default false)");
17
amdxdna_iommu_alloc_iova(struct amdxdna_dev * xdna,size_t size,dma_addr_t * dma_addr,bool size_aligned)18 static struct iova *amdxdna_iommu_alloc_iova(struct amdxdna_dev *xdna,
19 size_t size,
20 dma_addr_t *dma_addr,
21 bool size_aligned)
22 {
23 unsigned long shift, end;
24 struct iova *iova;
25
26 end = xdna->domain->geometry.aperture_end;
27 shift = iova_shift(&xdna->iovad);
28 size = iova_align(&xdna->iovad, size);
29
30 iova = alloc_iova(&xdna->iovad, size >> shift, end >> shift, size_aligned);
31 if (!iova)
32 return ERR_PTR(-ENOMEM);
33
34 *dma_addr = iova_dma_addr(&xdna->iovad, iova);
35
36 return iova;
37 }
38
amdxdna_dma_map_bo(struct amdxdna_dev * xdna,struct amdxdna_gem_obj * abo)39 int amdxdna_dma_map_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
40 {
41 unsigned long contig_sz;
42 struct sg_table *sgt;
43 dma_addr_t dma_addr;
44 struct iova *iova;
45 ssize_t size;
46
47 if (abo->type != AMDXDNA_BO_DEV_HEAP && abo->type != AMDXDNA_BO_SHARE)
48 return 0;
49
50 sgt = drm_gem_shmem_get_pages_sgt(&abo->base);
51 if (IS_ERR(sgt)) {
52 XDNA_ERR(xdna, "Get sgt failed, ret %ld", PTR_ERR(sgt));
53 return PTR_ERR(sgt);
54 }
55
56 if (!sgt->orig_nents) {
57 XDNA_ERR(xdna, "sgl is zero length");
58 return -EOPNOTSUPP;
59 }
60
61 if (amdxdna_iova_on(xdna)) {
62 if (!sg_page(sgt->sgl)) {
63 XDNA_ERR(xdna, "sgl is not page backed");
64 return -EOPNOTSUPP;
65 }
66
67 iova = amdxdna_iommu_alloc_iova(xdna, abo->mem.size, &dma_addr,
68 (abo->type == AMDXDNA_BO_DEV_HEAP));
69 if (IS_ERR(iova)) {
70 XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
71 return PTR_ERR(iova);
72 }
73
74 size = iommu_map_sgtable(xdna->domain, dma_addr, sgt,
75 IOMMU_READ | IOMMU_WRITE);
76 if (size < 0) {
77 XDNA_ERR(xdna, "iommu_map_sgtable failed: %zd", size);
78 __free_iova(&xdna->iovad, iova);
79 return size;
80 }
81 if (size < abo->mem.size) {
82 iommu_unmap(xdna->domain, dma_addr, size);
83 __free_iova(&xdna->iovad, iova);
84 return -ENXIO;
85 }
86 abo->mem.dma_addr = dma_addr;
87 } else {
88 /* Device doesn't support scatter/gather list, fail non-contiguous mapping. */
89 contig_sz = drm_prime_get_contiguous_size(sgt);
90 if (contig_sz < abo->mem.size) {
91 XDNA_ERR(xdna,
92 "noncontiguous dma addr, contig size:%ld, expected size:%ld",
93 contig_sz, abo->mem.size);
94 return -EINVAL;
95 }
96 abo->mem.dma_addr = sg_dma_address(sgt->sgl);
97 }
98 return 0;
99 }
100
amdxdna_dma_unmap_bo(struct amdxdna_dev * xdna,struct amdxdna_gem_obj * abo)101 void amdxdna_dma_unmap_bo(struct amdxdna_dev *xdna, struct amdxdna_gem_obj *abo)
102 {
103 size_t size;
104
105 if (abo->mem.dma_addr == AMDXDNA_INVALID_ADDR)
106 return;
107
108 if (amdxdna_iova_on(xdna)) {
109 size = iova_align(&xdna->iovad, abo->mem.size);
110 iommu_unmap(xdna->domain, abo->mem.dma_addr, size);
111 free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, abo->mem.dma_addr));
112 }
113 abo->mem.dma_addr = AMDXDNA_INVALID_ADDR;
114 }
115
amdxdna_iommu_alloc(struct amdxdna_dev * xdna,size_t size,dma_addr_t * dma_addr)116 void *amdxdna_iommu_alloc(struct amdxdna_dev *xdna, size_t size, dma_addr_t *dma_addr)
117 {
118 struct iova *iova;
119 void *cpu_addr;
120 int ret;
121
122 iova = amdxdna_iommu_alloc_iova(xdna, size, dma_addr, true);
123 if (IS_ERR(iova)) {
124 XDNA_ERR(xdna, "Alloc iova failed, ret %ld", PTR_ERR(iova));
125 return iova;
126 }
127
128 cpu_addr = (void *)__get_free_pages(GFP_KERNEL, get_order(size));
129 if (!cpu_addr) {
130 ret = -ENOMEM;
131 goto free_iova;
132 }
133
134 ret = iommu_map(xdna->domain, *dma_addr, virt_to_phys(cpu_addr),
135 iova_align(&xdna->iovad, size),
136 IOMMU_READ | IOMMU_WRITE, GFP_KERNEL);
137 if (ret)
138 goto free_cpu_addr;
139
140 return cpu_addr;
141
142 free_cpu_addr:
143 free_pages((unsigned long)cpu_addr, get_order(size));
144 free_iova:
145 __free_iova(&xdna->iovad, iova);
146 return ERR_PTR(ret);
147 }
148
amdxdna_iommu_free(struct amdxdna_dev * xdna,size_t size,void * cpu_addr,dma_addr_t dma_addr)149 void amdxdna_iommu_free(struct amdxdna_dev *xdna, size_t size,
150 void *cpu_addr, dma_addr_t dma_addr)
151 {
152 iommu_unmap(xdna->domain, dma_addr, iova_align(&xdna->iovad, size));
153 free_iova(&xdna->iovad, iova_pfn(&xdna->iovad, dma_addr));
154 free_pages((unsigned long)cpu_addr, get_order(size));
155 }
156
amdxdna_cleanup_force_iova(struct drm_device * dev,void * res)157 static void amdxdna_cleanup_force_iova(struct drm_device *dev, void *res)
158 {
159 struct amdxdna_dev *xdna = to_xdna_dev(dev);
160
161 if (xdna->domain) {
162 iommu_detach_group(xdna->domain, xdna->group);
163 put_iova_domain(&xdna->iovad);
164 iova_cache_put();
165 iommu_domain_free(xdna->domain);
166 }
167
168 iommu_group_put(xdna->group);
169 }
170
amdxdna_iommu_fini(struct amdxdna_dev * xdna)171 void amdxdna_iommu_fini(struct amdxdna_dev *xdna)
172 {
173 if (xdna->group && !xdna->domain)
174 iommu_group_put(xdna->group);
175 }
176
amdxdna_iommu_init(struct amdxdna_dev * xdna)177 int amdxdna_iommu_init(struct amdxdna_dev *xdna)
178 {
179 unsigned long order;
180 int ret = 0;
181
182 xdna->group = iommu_group_get(xdna->ddev.dev);
183 if (!xdna->group || !force_iova)
184 return 0;
185
186 XDNA_WARN(xdna, "Enabled force_iova mode.");
187 xdna->domain = iommu_paging_domain_alloc_flags(xdna->ddev.dev,
188 IOMMU_HWPT_ALLOC_PASID);
189 if (IS_ERR(xdna->domain)) {
190 XDNA_ERR(xdna, "Failed to alloc iommu domain");
191 ret = PTR_ERR(xdna->domain);
192 goto put_group;
193 }
194
195 ret = iova_cache_get();
196 if (ret)
197 goto free_domain;
198
199 order = __ffs(xdna->domain->pgsize_bitmap);
200 init_iova_domain(&xdna->iovad, 1UL << order, 0);
201
202 ret = iommu_attach_group(xdna->domain, xdna->group);
203 if (ret)
204 goto put_iova;
205
206 ret = drmm_add_action(&xdna->ddev, amdxdna_cleanup_force_iova, NULL);
207 if (ret)
208 goto detach_group;
209
210 return 0;
211
212 detach_group:
213 iommu_detach_group(xdna->domain, xdna->group);
214 put_iova:
215 put_iova_domain(&xdna->iovad);
216 iova_cache_put();
217 free_domain:
218 iommu_domain_free(xdna->domain);
219 put_group:
220 iommu_group_put(xdna->group);
221 xdna->group = NULL;
222 xdna->domain = NULL;
223
224 return ret;
225 }
226