1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * DMA BUF Mapping Helpers 4 * 5 */ 6 #include <linux/dma-buf-mapping.h> 7 #include <linux/dma-resv.h> 8 #include <linux/overflow.h> 9 #include <linux/align.h> 10 11 #define MAX_SG_ENT_SZ ALIGN_DOWN(UINT_MAX, PAGE_SIZE) 12 13 static struct scatterlist *fill_sg_entry(struct scatterlist *sgl, size_t length, 14 dma_addr_t addr) 15 { 16 size_t len; 17 18 while (length) { 19 len = min(length, MAX_SG_ENT_SZ); 20 length -= len; 21 /* 22 * DMABUF abuses scatterlist to create a scatterlist 23 * that does not have any CPU list, only the DMA list. 24 * Always set the page related values to NULL to ensure 25 * importers can't use it. The phys_addr based DMA API 26 * does not require the CPU list for mapping or unmapping. 27 */ 28 sg_set_page(sgl, NULL, 0, 0); 29 sg_dma_address(sgl) = addr; 30 sg_dma_len(sgl) = len; 31 addr += len; 32 /* Unconditionally advance. On last segment, this becomes NULL */ 33 sgl = sg_next(sgl); 34 } 35 36 return sgl; 37 } 38 39 static unsigned int calc_sg_nents(struct dma_iova_state *state, 40 struct phys_vec *phys_vec, size_t nr_ranges, 41 size_t size) 42 { 43 unsigned int nents = 0; 44 size_t i; 45 46 if (!state || !dma_use_iova(state)) { 47 for (i = 0; i < nr_ranges; i++) { 48 unsigned int added = DIV_ROUND_UP(phys_vec[i].len, MAX_SG_ENT_SZ); 49 50 if (check_add_overflow(nents, added, &nents)) 51 return 0; 52 } 53 } else { 54 /* 55 * In IOVA case, there is only one SG entry which spans 56 * for whole IOVA address space, but we need to make sure 57 * that it fits sg->length, maybe we need more. 58 */ 59 nents = DIV_ROUND_UP(size, MAX_SG_ENT_SZ); 60 } 61 62 return nents; 63 } 64 65 /** 66 * struct dma_buf_dma - holds DMA mapping information 67 * @sgt: Scatter-gather table 68 * @state: DMA IOVA state relevant in IOMMU-based DMA 69 * @size: Total size of DMA transfer 70 */ 71 struct dma_buf_dma { 72 struct sg_table sgt; 73 struct dma_iova_state *state; 74 size_t size; 75 }; 76 77 /** 78 * dma_buf_phys_vec_to_sgt - Returns the scatterlist table of the attachment 79 * from arrays of physical vectors. This funciton is intended for MMIO memory 80 * only. 81 * @attach: [in] attachment whose scatterlist is to be returned 82 * @provider: [in] p2pdma provider 83 * @phys_vec: [in] array of physical vectors 84 * @nr_ranges: [in] number of entries in phys_vec array 85 * @size: [in] total size of phys_vec 86 * @dir: [in] direction of DMA transfer 87 * 88 * Returns sg_table containing the scatterlist to be returned; returns ERR_PTR 89 * on error. May return -EINTR if it is interrupted by a signal. 90 * 91 * On success, the DMA addresses and lengths in the returned scatterlist are 92 * PAGE_SIZE aligned. 93 * 94 * A mapping must be unmapped by using dma_buf_free_sgt(). 95 * 96 * NOTE: This function is intended for exporters. If direct traffic routing is 97 * mandatory exporter should call routing pci_p2pdma_map_type() before calling 98 * this function. 99 */ 100 struct sg_table *dma_buf_phys_vec_to_sgt(struct dma_buf_attachment *attach, 101 struct p2pdma_provider *provider, 102 struct phys_vec *phys_vec, 103 size_t nr_ranges, size_t size, 104 enum dma_data_direction dir) 105 { 106 struct dma_buf_dma *dma; 107 struct scatterlist *sgl; 108 size_t mapped_len = 0; 109 unsigned int nents; 110 dma_addr_t addr; 111 size_t i; 112 int ret; 113 114 dma_resv_assert_held(attach->dmabuf->resv); 115 116 if (WARN_ON(!attach || !attach->dmabuf || !provider)) 117 /* This function is supposed to work on MMIO memory only */ 118 return ERR_PTR(-EINVAL); 119 120 dma = kzalloc_obj(*dma); 121 if (!dma) 122 return ERR_PTR(-ENOMEM); 123 124 switch (pci_p2pdma_map_type(provider, attach->dev)) { 125 case PCI_P2PDMA_MAP_BUS_ADDR: 126 /* 127 * There is no need in IOVA at all for this flow. 128 */ 129 break; 130 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 131 dma->state = kzalloc_obj(*dma->state); 132 if (!dma->state) { 133 ret = -ENOMEM; 134 goto err_free_dma; 135 } 136 137 dma_iova_try_alloc(attach->dev, dma->state, 0, size); 138 break; 139 default: 140 ret = -EINVAL; 141 goto err_free_dma; 142 } 143 144 nents = calc_sg_nents(dma->state, phys_vec, nr_ranges, size); 145 146 /* sg_alloc_table will cleanly fail and return -EINVAL if nents == 0 */ 147 ret = sg_alloc_table(&dma->sgt, nents, GFP_KERNEL | __GFP_ZERO); 148 if (ret) 149 goto err_free_state; 150 151 sgl = dma->sgt.sgl; 152 153 for (i = 0; i < nr_ranges; i++) { 154 if (!dma->state) { 155 addr = pci_p2pdma_bus_addr_map(provider, 156 phys_vec[i].paddr); 157 } else if (dma_use_iova(dma->state)) { 158 ret = dma_iova_link(attach->dev, dma->state, 159 phys_vec[i].paddr, 0, 160 phys_vec[i].len, dir, 161 DMA_ATTR_MMIO); 162 if (ret) 163 goto err_unmap_dma; 164 165 mapped_len += phys_vec[i].len; 166 } else { 167 addr = dma_map_phys(attach->dev, phys_vec[i].paddr, 168 phys_vec[i].len, dir, 169 DMA_ATTR_MMIO); 170 ret = dma_mapping_error(attach->dev, addr); 171 if (ret) 172 goto err_unmap_dma; 173 } 174 175 if (!dma->state || !dma_use_iova(dma->state)) 176 sgl = fill_sg_entry(sgl, phys_vec[i].len, addr); 177 } 178 179 if (dma->state && dma_use_iova(dma->state)) { 180 WARN_ON_ONCE(mapped_len != size); 181 ret = dma_iova_sync(attach->dev, dma->state, 0, mapped_len); 182 if (ret) 183 goto err_unmap_dma; 184 185 sgl = fill_sg_entry(sgl, mapped_len, dma->state->addr); 186 } 187 188 dma->size = size; 189 190 /* 191 * No CPU list included — set orig_nents = 0 so others can detect 192 * this via SG table (use nents only). 193 */ 194 dma->sgt.orig_nents = 0; 195 196 197 /* 198 * SGL must be NULL to indicate that SGL is the last one 199 * and we allocated correct number of entries in sg_alloc_table() 200 */ 201 WARN_ON_ONCE(sgl); 202 return &dma->sgt; 203 204 err_unmap_dma: 205 if (!i || !dma->state) { 206 ; /* Do nothing */ 207 } else if (dma_use_iova(dma->state)) { 208 dma_iova_destroy(attach->dev, dma->state, mapped_len, dir, 209 DMA_ATTR_MMIO); 210 } else { 211 for_each_sgtable_dma_sg(&dma->sgt, sgl, i) 212 dma_unmap_phys(attach->dev, sg_dma_address(sgl), 213 sg_dma_len(sgl), dir, DMA_ATTR_MMIO); 214 } 215 sg_free_table(&dma->sgt); 216 err_free_state: 217 kfree(dma->state); 218 err_free_dma: 219 kfree(dma); 220 return ERR_PTR(ret); 221 } 222 EXPORT_SYMBOL_NS_GPL(dma_buf_phys_vec_to_sgt, "DMA_BUF"); 223 224 /** 225 * dma_buf_free_sgt- unmaps the buffer 226 * @attach: [in] attachment to unmap buffer from 227 * @sgt: [in] scatterlist info of the buffer to unmap 228 * @dir: [in] direction of DMA transfer 229 * 230 * This unmaps a DMA mapping for @attached obtained 231 * by dma_buf_phys_vec_to_sgt(). 232 */ 233 void dma_buf_free_sgt(struct dma_buf_attachment *attach, struct sg_table *sgt, 234 enum dma_data_direction dir) 235 { 236 struct dma_buf_dma *dma = container_of(sgt, struct dma_buf_dma, sgt); 237 int i; 238 239 dma_resv_assert_held(attach->dmabuf->resv); 240 241 if (!dma->state) { 242 ; /* Do nothing */ 243 } else if (dma_use_iova(dma->state)) { 244 dma_iova_destroy(attach->dev, dma->state, dma->size, dir, 245 DMA_ATTR_MMIO); 246 } else { 247 struct scatterlist *sgl; 248 249 for_each_sgtable_dma_sg(sgt, sgl, i) 250 dma_unmap_phys(attach->dev, sg_dma_address(sgl), 251 sg_dma_len(sgl), dir, DMA_ATTR_MMIO); 252 } 253 254 sg_free_table(sgt); 255 kfree(dma->state); 256 kfree(dma); 257 258 } 259 EXPORT_SYMBOL_NS_GPL(dma_buf_free_sgt, "DMA_BUF"); 260