1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2026, Advanced Micro Devices, Inc. 4 */ 5 6 #include <drm/drm_mm.h> 7 #include <drm/drm_prime.h> 8 9 #include "amdxdna_cbuf.h" 10 #include "amdxdna_pci_drv.h" 11 12 /* 13 * Carveout memory is a chunk of memory which is physically contiguous and 14 * is reserved during early boot time. There is only one chunk of such memory 15 * per device. Once available, all BOs accessible from device should be 16 * allocated from this memory. This is a platform debug/bringup feature. 17 */ 18 struct amdxdna_carveout { 19 u64 addr; 20 u64 size; 21 struct drm_mm mm; 22 struct mutex lock; /* protect mm */ 23 }; 24 25 bool amdxdna_use_carveout(struct amdxdna_dev *xdna) 26 { 27 return !!xdna->carveout; 28 } 29 30 void amdxdna_get_carveout_conf(struct amdxdna_dev *xdna, u64 *addr, u64 *size) 31 { 32 if (amdxdna_use_carveout(xdna)) { 33 *addr = xdna->carveout->addr; 34 *size = xdna->carveout->size; 35 } else { 36 *addr = 0; 37 *size = 0; 38 } 39 } 40 41 int amdxdna_carveout_init(struct amdxdna_dev *xdna, u64 carveout_addr, u64 carveout_size) 42 { 43 struct amdxdna_carveout *carveout; 44 45 /* Only allow carveout memory to be set up once. */ 46 if (amdxdna_use_carveout(xdna)) { 47 XDNA_ERR(xdna, "Carveout memory has already been set up."); 48 return -EBUSY; 49 } 50 51 carveout = kzalloc_obj(*carveout); 52 if (!carveout) 53 return -ENOMEM; 54 55 carveout->addr = carveout_addr; 56 carveout->size = carveout_size; 57 mutex_init(&carveout->lock); 58 drm_mm_init(&carveout->mm, carveout->addr, carveout->size); 59 60 xdna->carveout = carveout; 61 XDNA_INFO(xdna, "Use carveout mem: 0x%llx@0x%llx\n", carveout->size, carveout->addr); 62 return 0; 63 } 64 65 void amdxdna_carveout_fini(struct amdxdna_dev *xdna) 66 { 67 struct amdxdna_carveout *carveout = xdna->carveout; 68 69 if (!amdxdna_use_carveout(xdna)) 70 return; 71 72 XDNA_INFO(xdna, "Cleanup carveout mem: 0x%llx@0x%llx\n", carveout->size, carveout->addr); 73 drm_mm_takedown(&carveout->mm); 74 mutex_destroy(&carveout->lock); 75 kfree(carveout); 76 xdna->carveout = NULL; 77 } 78 79 struct amdxdna_cbuf_priv { 80 struct amdxdna_dev *xdna; 81 struct drm_mm_node node; 82 }; 83 84 static struct sg_table *amdxdna_cbuf_map(struct dma_buf_attachment *attach, 85 enum dma_data_direction direction) 86 { 87 struct amdxdna_cbuf_priv *cbuf = attach->dmabuf->priv; 88 struct device *dev = attach->dev; 89 struct scatterlist *sgl, *sg; 90 int ret, n_entries, i; 91 struct sg_table *sgt; 92 dma_addr_t dma_addr; 93 size_t dma_size; 94 size_t max_seg; 95 96 sgt = kzalloc_obj(*sgt); 97 if (!sgt) 98 return ERR_PTR(-ENOMEM); 99 100 max_seg = min_t(size_t, UINT_MAX, dma_max_mapping_size(dev)); 101 n_entries = (cbuf->node.size + max_seg - 1) / max_seg; 102 sgl = kzalloc_objs(*sg, n_entries); 103 if (!sgl) { 104 ret = -ENOMEM; 105 goto free_sgt; 106 } 107 sg_init_table(sgl, n_entries); 108 sgt->orig_nents = n_entries; 109 sgt->nents = n_entries; 110 sgt->sgl = sgl; 111 112 dma_size = cbuf->node.size; 113 dma_addr = dma_map_resource(dev, cbuf->node.start, dma_size, 114 direction, DMA_ATTR_SKIP_CPU_SYNC); 115 ret = dma_mapping_error(dev, dma_addr); 116 if (ret) { 117 pr_err("Failed to dma_map_resource carveout dma buf, ret %d\n", ret); 118 goto free_sgl; 119 } 120 121 for_each_sgtable_dma_sg(sgt, sg, i) { 122 size_t len = min_t(size_t, max_seg, dma_size); 123 124 sg_dma_address(sg) = dma_addr; 125 sg_dma_len(sg) = len; 126 dma_addr += len; 127 dma_size -= len; 128 } 129 130 return sgt; 131 132 free_sgl: 133 kfree(sgl); 134 free_sgt: 135 kfree(sgt); 136 return ERR_PTR(ret); 137 } 138 139 static void amdxdna_cbuf_unmap(struct dma_buf_attachment *attach, 140 struct sg_table *sgt, 141 enum dma_data_direction direction) 142 { 143 dma_unmap_resource(attach->dev, sg_dma_address(sgt->sgl), 144 drm_prime_get_contiguous_size(sgt), direction, 145 DMA_ATTR_SKIP_CPU_SYNC); 146 sg_free_table(sgt); 147 kfree(sgt); 148 } 149 150 static void amdxdna_cbuf_release(struct dma_buf *dbuf) 151 { 152 struct amdxdna_cbuf_priv *cbuf = dbuf->priv; 153 struct amdxdna_carveout *carveout; 154 155 carveout = cbuf->xdna->carveout; 156 mutex_lock(&carveout->lock); 157 drm_mm_remove_node(&cbuf->node); 158 mutex_unlock(&carveout->lock); 159 160 kfree(cbuf); 161 } 162 163 static vm_fault_t amdxdna_cbuf_vm_fault(struct vm_fault *vmf) 164 { 165 struct vm_area_struct *vma = vmf->vma; 166 struct amdxdna_cbuf_priv *cbuf; 167 unsigned long pfn; 168 pgoff_t pgoff; 169 170 cbuf = vma->vm_private_data; 171 pgoff = (vmf->address - vma->vm_start) >> PAGE_SHIFT; 172 pfn = (cbuf->node.start >> PAGE_SHIFT) + pgoff; 173 174 return vmf_insert_pfn(vma, vmf->address, pfn); 175 } 176 177 static const struct vm_operations_struct amdxdna_cbuf_vm_ops = { 178 .fault = amdxdna_cbuf_vm_fault, 179 }; 180 181 static int amdxdna_cbuf_mmap(struct dma_buf *dbuf, struct vm_area_struct *vma) 182 { 183 struct amdxdna_cbuf_priv *cbuf = dbuf->priv; 184 185 vma->vm_ops = &amdxdna_cbuf_vm_ops; 186 vma->vm_private_data = cbuf; 187 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP); 188 189 return 0; 190 } 191 192 static int amdxdna_cbuf_vmap(struct dma_buf *dbuf, struct iosys_map *map) 193 { 194 struct amdxdna_cbuf_priv *cbuf = dbuf->priv; 195 void *kva; 196 197 kva = memremap(cbuf->node.start, cbuf->node.size, MEMREMAP_WB); 198 if (!kva) { 199 pr_err("Failed to vmap carveout dma buf\n"); 200 return -ENOMEM; 201 } 202 203 iosys_map_set_vaddr(map, kva); 204 return 0; 205 } 206 207 static void amdxdna_cbuf_vunmap(struct dma_buf *dbuf, struct iosys_map *map) 208 { 209 memunmap(map->vaddr); 210 } 211 212 static const struct dma_buf_ops amdxdna_cbuf_dmabuf_ops = { 213 .map_dma_buf = amdxdna_cbuf_map, 214 .unmap_dma_buf = amdxdna_cbuf_unmap, 215 .release = amdxdna_cbuf_release, 216 .mmap = amdxdna_cbuf_mmap, 217 .vmap = amdxdna_cbuf_vmap, 218 .vunmap = amdxdna_cbuf_vunmap, 219 }; 220 221 static int amdxdna_cbuf_clear(struct dma_buf *dbuf) 222 { 223 struct iosys_map vmap = IOSYS_MAP_INIT_VADDR(NULL); 224 225 dma_buf_vmap(dbuf, &vmap); 226 if (!vmap.vaddr) 227 return -EFAULT; 228 229 memset(vmap.vaddr, 0, dbuf->size); 230 dma_buf_vunmap(dbuf, &vmap); 231 232 return 0; 233 } 234 235 struct dma_buf *amdxdna_get_cbuf(struct drm_device *dev, size_t size, u64 alignment) 236 { 237 struct amdxdna_dev *xdna = to_xdna_dev(dev); 238 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 239 struct amdxdna_carveout *carveout; 240 struct amdxdna_cbuf_priv *cbuf; 241 struct dma_buf *dbuf; 242 int ret; 243 244 cbuf = kzalloc_obj(*cbuf); 245 if (!cbuf) 246 return ERR_PTR(-ENOMEM); 247 cbuf->xdna = xdna; 248 249 carveout = xdna->carveout; 250 mutex_lock(&carveout->lock); 251 ret = drm_mm_insert_node_generic(&carveout->mm, &cbuf->node, size, 252 alignment, 0, DRM_MM_INSERT_BEST); 253 mutex_unlock(&carveout->lock); 254 if (ret) 255 goto free_cbuf; 256 257 exp_info.size = size; 258 exp_info.ops = &amdxdna_cbuf_dmabuf_ops; 259 exp_info.priv = cbuf; 260 exp_info.flags = O_RDWR; 261 dbuf = dma_buf_export(&exp_info); 262 if (IS_ERR(dbuf)) { 263 ret = PTR_ERR(dbuf); 264 goto remove_node; 265 } 266 267 ret = amdxdna_cbuf_clear(dbuf); 268 if (ret) { 269 dma_buf_put(dbuf); 270 goto out; 271 } 272 return dbuf; 273 274 remove_node: 275 drm_mm_remove_node(&cbuf->node); 276 free_cbuf: 277 kfree(cbuf); 278 out: 279 return ERR_PTR(ret); 280 } 281