1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * DMA BUF Mapping Helpers
4 *
5 */
6 #include <linux/dma-buf-mapping.h>
7 #include <linux/dma-resv.h>
8 #include <linux/overflow.h>
9 #include <linux/align.h>
10
11 #define MAX_SG_ENT_SZ ALIGN_DOWN(UINT_MAX, PAGE_SIZE)
12
fill_sg_entry(struct scatterlist * sgl,size_t length,dma_addr_t addr)13 static struct scatterlist *fill_sg_entry(struct scatterlist *sgl, size_t length,
14 dma_addr_t addr)
15 {
16 size_t len;
17
18 while (length) {
19 len = min(length, MAX_SG_ENT_SZ);
20 length -= len;
21 /*
22 * DMABUF abuses scatterlist to create a scatterlist
23 * that does not have any CPU list, only the DMA list.
24 * Always set the page related values to NULL to ensure
25 * importers can't use it. The phys_addr based DMA API
26 * does not require the CPU list for mapping or unmapping.
27 */
28 sg_set_page(sgl, NULL, 0, 0);
29 sg_dma_address(sgl) = addr;
30 sg_dma_len(sgl) = len;
31 addr += len;
32 /* Unconditionally advance. On last segment, this becomes NULL */
33 sgl = sg_next(sgl);
34 }
35
36 return sgl;
37 }
38
calc_sg_nents(struct dma_iova_state * state,struct phys_vec * phys_vec,size_t nr_ranges,size_t size)39 static unsigned int calc_sg_nents(struct dma_iova_state *state,
40 struct phys_vec *phys_vec, size_t nr_ranges,
41 size_t size)
42 {
43 unsigned int nents = 0;
44 size_t i;
45
46 if (!state || !dma_use_iova(state)) {
47 for (i = 0; i < nr_ranges; i++) {
48 unsigned int added = DIV_ROUND_UP(phys_vec[i].len, MAX_SG_ENT_SZ);
49
50 if (check_add_overflow(nents, added, &nents))
51 return 0;
52 }
53 } else {
54 /*
55 * In IOVA case, there is only one SG entry which spans
56 * for whole IOVA address space, but we need to make sure
57 * that it fits sg->length, maybe we need more.
58 */
59 nents = DIV_ROUND_UP(size, MAX_SG_ENT_SZ);
60 }
61
62 return nents;
63 }
64
65 /**
66 * struct dma_buf_dma - holds DMA mapping information
67 * @sgt: Scatter-gather table
68 * @state: DMA IOVA state relevant in IOMMU-based DMA
69 * @size: Total size of DMA transfer
70 */
71 struct dma_buf_dma {
72 struct sg_table sgt;
73 struct dma_iova_state *state;
74 size_t size;
75 };
76
77 /**
78 * dma_buf_phys_vec_to_sgt - Returns the scatterlist table of the attachment
79 * from arrays of physical vectors. This funciton is intended for MMIO memory
80 * only.
81 * @attach: [in] attachment whose scatterlist is to be returned
82 * @provider: [in] p2pdma provider
83 * @phys_vec: [in] array of physical vectors
84 * @nr_ranges: [in] number of entries in phys_vec array
85 * @size: [in] total size of phys_vec
86 * @dir: [in] direction of DMA transfer
87 *
88 * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR
89 * on error. May return -EINTR if it is interrupted by a signal.
90 *
91 * On success, the DMA addresses and lengths in the returned scatterlist are
92 * PAGE_SIZE aligned.
93 *
94 * A mapping must be unmapped by using dma_buf_free_sgt().
95 *
96 * NOTE: This function is intended for exporters. If direct traffic routing is
97 * mandatory exporter should call routing pci_p2pdma_map_type() before calling
98 * this function.
99 */
dma_buf_phys_vec_to_sgt(struct dma_buf_attachment * attach,struct p2pdma_provider * provider,struct phys_vec * phys_vec,size_t nr_ranges,size_t size,enum dma_data_direction dir)100 struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach,
101 struct p2pdma_provider *provider,
102 struct phys_vec *phys_vec,
103 size_t nr_ranges, size_t size,
104 enum dma_data_direction dir)
105 {
106 struct dma_buf_dma *dma;
107 struct scatterlist *sgl;
108 size_t mapped_len = 0;
109 unsigned int nents;
110 dma_addr_t addr;
111 size_t i;
112 int ret;
113
114 dma_resv_assert_held(attach->dmabuf->resv);
115
116 if (WARN_ON(!attach || !attach->dmabuf || !provider))
117 /* This function is supposed to work on MMIO memory only */
118 return ERR_PTR(-EINVAL);
119
120 dma = kzalloc_obj(*dma);
121 if (!dma)
122 return ERR_PTR(-ENOMEM);
123
124 switch (pci_p2pdma_map_type(provider, attach->dev)) {
125 case PCI_P2PDMA_MAP_BUS_ADDR:
126 /*
127 * There is no need in IOVA at all for this flow.
128 */
129 break;
130 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
131 dma->state = kzalloc_obj(*dma->state);
132 if (!dma->state) {
133 ret = -ENOMEM;
134 goto err_free_dma;
135 }
136
137 dma_iova_try_alloc(attach->dev, dma->state, 0, size);
138 break;
139 default:
140 ret = -EINVAL;
141 goto err_free_dma;
142 }
143
144 nents = calc_sg_nents(dma->state, phys_vec, nr_ranges, size);
145
146 /* sg_alloc_table will cleanly fail and return -EINVAL if nents == 0 */
147 ret = sg_alloc_table(&dma->sgt, nents, GFP_KERNEL | __GFP_ZERO);
148 if (ret)
149 goto err_free_state;
150
151 sgl = dma->sgt.sgl;
152
153 for (i = 0; i < nr_ranges; i++) {
154 if (!dma->state) {
155 addr = pci_p2pdma_bus_addr_map(provider,
156 phys_vec[i].paddr);
157 } else if (dma_use_iova(dma->state)) {
158 ret = dma_iova_link(attach->dev, dma->state,
159 phys_vec[i].paddr, 0,
160 phys_vec[i].len, dir,
161 DMA_ATTR_MMIO);
162 if (ret)
163 goto err_unmap_dma;
164
165 mapped_len += phys_vec[i].len;
166 } else {
167 addr = dma_map_phys(attach->dev, phys_vec[i].paddr,
168 phys_vec[i].len, dir,
169 DMA_ATTR_MMIO);
170 ret = dma_mapping_error(attach->dev, addr);
171 if (ret)
172 goto err_unmap_dma;
173 }
174
175 if (!dma->state || !dma_use_iova(dma->state))
176 sgl = fill_sg_entry(sgl, phys_vec[i].len, addr);
177 }
178
179 if (dma->state && dma_use_iova(dma->state)) {
180 WARN_ON_ONCE(mapped_len != size);
181 ret = dma_iova_sync(attach->dev, dma->state, 0, mapped_len);
182 if (ret)
183 goto err_unmap_dma;
184
185 sgl = fill_sg_entry(sgl, mapped_len, dma->state->addr);
186 }
187
188 dma->size = size;
189
190 /*
191 * No CPU list included — set orig_nents = 0 so others can detect
192 * this via SG table (use nents only).
193 */
194 dma->sgt.orig_nents = 0;
195
196
197 /*
198 * SGL must be NULL to indicate that SGL is the last one
199 * and we allocated correct number of entries in sg_alloc_table()
200 */
201 WARN_ON_ONCE(sgl);
202 return &dma->sgt;
203
204 err_unmap_dma:
205 if (!i || !dma->state) {
206 ; /* Do nothing */
207 } else if (dma_use_iova(dma->state)) {
208 dma_iova_destroy(attach->dev, dma->state, mapped_len, dir,
209 DMA_ATTR_MMIO);
210 } else {
211 for_each_sgtable_dma_sg(&dma->sgt, sgl, i)
212 dma_unmap_phys(attach->dev, sg_dma_address(sgl),
213 sg_dma_len(sgl), dir, DMA_ATTR_MMIO);
214 }
215 sg_free_table(&dma->sgt);
216 err_free_state:
217 kfree(dma->state);
218 err_free_dma:
219 kfree(dma);
220 return ERR_PTR(ret);
221 }
222 EXPORT_SYMBOL_NS_GPL(dma_buf_phys_vec_to_sgt, "DMA_BUF");
223
224 /**
225 * dma_buf_free_sgt- unmaps the buffer
226 * @attach: [in] attachment to unmap buffer from
227 * @sgt: [in] scatterlist info of the buffer to unmap
228 * @dir: [in] direction of DMA transfer
229 *
230 * This unmaps a DMA mapping for @attached obtained
231 * by dma_buf_phys_vec_to_sgt().
232 */
dma_buf_free_sgt(struct dma_buf_attachment * attach,struct sg_table * sgt,enum dma_data_direction dir)233 void dma_buf_free_sgt(struct dma_buf_attachment *attach, struct sg_table *sgt,
234 enum dma_data_direction dir)
235 {
236 struct dma_buf_dma *dma = container_of(sgt, struct dma_buf_dma, sgt);
237 int i;
238
239 dma_resv_assert_held(attach->dmabuf->resv);
240
241 if (!dma->state) {
242 ; /* Do nothing */
243 } else if (dma_use_iova(dma->state)) {
244 dma_iova_destroy(attach->dev, dma->state, dma->size, dir,
245 DMA_ATTR_MMIO);
246 } else {
247 struct scatterlist *sgl;
248
249 for_each_sgtable_dma_sg(sgt, sgl, i)
250 dma_unmap_phys(attach->dev, sg_dma_address(sgl),
251 sg_dma_len(sgl), dir, DMA_ATTR_MMIO);
252 }
253
254 sg_free_table(sgt);
255 kfree(dma->state);
256 kfree(dma);
257
258 }
259 EXPORT_SYMBOL_NS_GPL(dma_buf_free_sgt, "DMA_BUF");
260