1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * DMABUF System heap exporter 4 * 5 * Copyright (C) 2011 Google, Inc. 6 * Copyright (C) 2019, 2020 Linaro Ltd. 7 * 8 * Portions based off of Andrew Davis' SRAM heap: 9 * Copyright (C) 2019 Texas Instruments Incorporated - http://www.ti.com/ 10 * Andrew F. Davis <afd@ti.com> 11 */ 12 13 #include <linux/dma-buf.h> 14 #include <linux/dma-mapping.h> 15 #include <linux/dma-heap.h> 16 #include <linux/err.h> 17 #include <linux/highmem.h> 18 #include <linux/mm.h> 19 #include <linux/module.h> 20 #include <linux/scatterlist.h> 21 #include <linux/slab.h> 22 #include <linux/vmalloc.h> 23 24 struct system_heap_buffer { 25 struct dma_heap *heap; 26 struct list_head attachments; 27 struct mutex lock; 28 unsigned long len; 29 struct sg_table sg_table; 30 int vmap_cnt; 31 void *vaddr; 32 }; 33 34 struct dma_heap_attachment { 35 struct device *dev; 36 struct sg_table table; 37 struct list_head list; 38 bool mapped; 39 }; 40 41 #define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO) 42 #define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ 43 | __GFP_NORETRY) & ~__GFP_RECLAIM) \ 44 | __GFP_COMP) 45 static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; 46 /* 47 * The selection of the orders used for allocation (1MB, 64K, 4K) is designed 48 * to match with the sizes often found in IOMMUs. Using order 4 pages instead 49 * of order 0 pages can significantly improve the performance of many IOMMUs 50 * by reducing TLB pressure and time spent updating page tables. 51 */ 52 static const unsigned int orders[] = {8, 4, 0}; 53 #define NUM_ORDERS ARRAY_SIZE(orders) 54 55 static int dup_sg_table(struct sg_table *from, struct sg_table *to) 56 { 57 struct scatterlist *sg, *new_sg; 58 int ret, i; 59 60 ret = sg_alloc_table(to, from->orig_nents, GFP_KERNEL); 61 if (ret) 62 return ret; 63 64 new_sg = to->sgl; 65 for_each_sgtable_sg(from, sg, i) { 66 sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset); 67 new_sg = sg_next(new_sg); 68 } 69 70 return 0; 71 } 72 73 static int system_heap_attach(struct dma_buf *dmabuf, 74 struct dma_buf_attachment *attachment) 75 { 76 struct system_heap_buffer *buffer = dmabuf->priv; 77 struct dma_heap_attachment *a; 78 int ret; 79 80 a = kzalloc(sizeof(*a), GFP_KERNEL); 81 if (!a) 82 return -ENOMEM; 83 84 ret = dup_sg_table(&buffer->sg_table, &a->table); 85 if (ret) { 86 kfree(a); 87 return ret; 88 } 89 90 a->dev = attachment->dev; 91 INIT_LIST_HEAD(&a->list); 92 a->mapped = false; 93 94 attachment->priv = a; 95 96 mutex_lock(&buffer->lock); 97 list_add(&a->list, &buffer->attachments); 98 mutex_unlock(&buffer->lock); 99 100 return 0; 101 } 102 103 static void system_heap_detach(struct dma_buf *dmabuf, 104 struct dma_buf_attachment *attachment) 105 { 106 struct system_heap_buffer *buffer = dmabuf->priv; 107 struct dma_heap_attachment *a = attachment->priv; 108 109 mutex_lock(&buffer->lock); 110 list_del(&a->list); 111 mutex_unlock(&buffer->lock); 112 113 sg_free_table(&a->table); 114 kfree(a); 115 } 116 117 static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment, 118 enum dma_data_direction direction) 119 { 120 struct dma_heap_attachment *a = attachment->priv; 121 struct sg_table *table = &a->table; 122 int ret; 123 124 ret = dma_map_sgtable(attachment->dev, table, direction, 0); 125 if (ret) 126 return ERR_PTR(ret); 127 128 a->mapped = true; 129 return table; 130 } 131 132 static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, 133 struct sg_table *table, 134 enum dma_data_direction direction) 135 { 136 struct dma_heap_attachment *a = attachment->priv; 137 138 a->mapped = false; 139 dma_unmap_sgtable(attachment->dev, table, direction, 0); 140 } 141 142 static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, 143 enum dma_data_direction direction) 144 { 145 struct system_heap_buffer *buffer = dmabuf->priv; 146 struct dma_heap_attachment *a; 147 148 mutex_lock(&buffer->lock); 149 150 if (buffer->vmap_cnt) 151 invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); 152 153 list_for_each_entry(a, &buffer->attachments, list) { 154 if (!a->mapped) 155 continue; 156 dma_sync_sgtable_for_cpu(a->dev, &a->table, direction); 157 } 158 mutex_unlock(&buffer->lock); 159 160 return 0; 161 } 162 163 static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, 164 enum dma_data_direction direction) 165 { 166 struct system_heap_buffer *buffer = dmabuf->priv; 167 struct dma_heap_attachment *a; 168 169 mutex_lock(&buffer->lock); 170 171 if (buffer->vmap_cnt) 172 flush_kernel_vmap_range(buffer->vaddr, buffer->len); 173 174 list_for_each_entry(a, &buffer->attachments, list) { 175 if (!a->mapped) 176 continue; 177 dma_sync_sgtable_for_device(a->dev, &a->table, direction); 178 } 179 mutex_unlock(&buffer->lock); 180 181 return 0; 182 } 183 184 static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) 185 { 186 struct system_heap_buffer *buffer = dmabuf->priv; 187 struct sg_table *table = &buffer->sg_table; 188 unsigned long addr = vma->vm_start; 189 struct sg_page_iter piter; 190 int ret; 191 192 for_each_sgtable_page(table, &piter, vma->vm_pgoff) { 193 struct page *page = sg_page_iter_page(&piter); 194 195 ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, 196 vma->vm_page_prot); 197 if (ret) 198 return ret; 199 addr += PAGE_SIZE; 200 if (addr >= vma->vm_end) 201 return 0; 202 } 203 return 0; 204 } 205 206 static void *system_heap_do_vmap(struct system_heap_buffer *buffer) 207 { 208 struct sg_table *table = &buffer->sg_table; 209 int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; 210 struct page **pages = vmalloc(sizeof(struct page *) * npages); 211 struct page **tmp = pages; 212 struct sg_page_iter piter; 213 void *vaddr; 214 215 if (!pages) 216 return ERR_PTR(-ENOMEM); 217 218 for_each_sgtable_page(table, &piter, 0) { 219 WARN_ON(tmp - pages >= npages); 220 *tmp++ = sg_page_iter_page(&piter); 221 } 222 223 vaddr = vmap(pages, npages, VM_MAP, PAGE_KERNEL); 224 vfree(pages); 225 226 if (!vaddr) 227 return ERR_PTR(-ENOMEM); 228 229 return vaddr; 230 } 231 232 static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map) 233 { 234 struct system_heap_buffer *buffer = dmabuf->priv; 235 void *vaddr; 236 int ret = 0; 237 238 mutex_lock(&buffer->lock); 239 if (buffer->vmap_cnt) { 240 buffer->vmap_cnt++; 241 iosys_map_set_vaddr(map, buffer->vaddr); 242 goto out; 243 } 244 245 vaddr = system_heap_do_vmap(buffer); 246 if (IS_ERR(vaddr)) { 247 ret = PTR_ERR(vaddr); 248 goto out; 249 } 250 251 buffer->vaddr = vaddr; 252 buffer->vmap_cnt++; 253 iosys_map_set_vaddr(map, buffer->vaddr); 254 out: 255 mutex_unlock(&buffer->lock); 256 257 return ret; 258 } 259 260 static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) 261 { 262 struct system_heap_buffer *buffer = dmabuf->priv; 263 264 mutex_lock(&buffer->lock); 265 if (!--buffer->vmap_cnt) { 266 vunmap(buffer->vaddr); 267 buffer->vaddr = NULL; 268 } 269 mutex_unlock(&buffer->lock); 270 iosys_map_clear(map); 271 } 272 273 static void system_heap_dma_buf_release(struct dma_buf *dmabuf) 274 { 275 struct system_heap_buffer *buffer = dmabuf->priv; 276 struct sg_table *table; 277 struct scatterlist *sg; 278 int i; 279 280 table = &buffer->sg_table; 281 for_each_sgtable_sg(table, sg, i) { 282 struct page *page = sg_page(sg); 283 284 __free_pages(page, compound_order(page)); 285 } 286 sg_free_table(table); 287 kfree(buffer); 288 } 289 290 static const struct dma_buf_ops system_heap_buf_ops = { 291 .attach = system_heap_attach, 292 .detach = system_heap_detach, 293 .map_dma_buf = system_heap_map_dma_buf, 294 .unmap_dma_buf = system_heap_unmap_dma_buf, 295 .begin_cpu_access = system_heap_dma_buf_begin_cpu_access, 296 .end_cpu_access = system_heap_dma_buf_end_cpu_access, 297 .mmap = system_heap_mmap, 298 .vmap = system_heap_vmap, 299 .vunmap = system_heap_vunmap, 300 .release = system_heap_dma_buf_release, 301 }; 302 303 static struct page *alloc_largest_available(unsigned long size, 304 unsigned int max_order) 305 { 306 struct page *page; 307 int i; 308 309 for (i = 0; i < NUM_ORDERS; i++) { 310 if (size < (PAGE_SIZE << orders[i])) 311 continue; 312 if (max_order < orders[i]) 313 continue; 314 315 page = alloc_pages(order_flags[i], orders[i]); 316 if (!page) 317 continue; 318 return page; 319 } 320 return NULL; 321 } 322 323 static struct dma_buf *system_heap_allocate(struct dma_heap *heap, 324 unsigned long len, 325 u32 fd_flags, 326 u64 heap_flags) 327 { 328 struct system_heap_buffer *buffer; 329 DEFINE_DMA_BUF_EXPORT_INFO(exp_info); 330 unsigned long size_remaining = len; 331 unsigned int max_order = orders[0]; 332 struct dma_buf *dmabuf; 333 struct sg_table *table; 334 struct scatterlist *sg; 335 struct list_head pages; 336 struct page *page, *tmp_page; 337 int i, ret = -ENOMEM; 338 339 buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); 340 if (!buffer) 341 return ERR_PTR(-ENOMEM); 342 343 INIT_LIST_HEAD(&buffer->attachments); 344 mutex_init(&buffer->lock); 345 buffer->heap = heap; 346 buffer->len = len; 347 348 INIT_LIST_HEAD(&pages); 349 i = 0; 350 while (size_remaining > 0) { 351 /* 352 * Avoid trying to allocate memory if the process 353 * has been killed by SIGKILL 354 */ 355 if (fatal_signal_pending(current)) { 356 ret = -EINTR; 357 goto free_buffer; 358 } 359 360 page = alloc_largest_available(size_remaining, max_order); 361 if (!page) 362 goto free_buffer; 363 364 list_add_tail(&page->lru, &pages); 365 size_remaining -= page_size(page); 366 max_order = compound_order(page); 367 i++; 368 } 369 370 table = &buffer->sg_table; 371 if (sg_alloc_table(table, i, GFP_KERNEL)) 372 goto free_buffer; 373 374 sg = table->sgl; 375 list_for_each_entry_safe(page, tmp_page, &pages, lru) { 376 sg_set_page(sg, page, page_size(page), 0); 377 sg = sg_next(sg); 378 list_del(&page->lru); 379 } 380 381 /* create the dmabuf */ 382 exp_info.exp_name = dma_heap_get_name(heap); 383 exp_info.ops = &system_heap_buf_ops; 384 exp_info.size = buffer->len; 385 exp_info.flags = fd_flags; 386 exp_info.priv = buffer; 387 dmabuf = dma_buf_export(&exp_info); 388 if (IS_ERR(dmabuf)) { 389 ret = PTR_ERR(dmabuf); 390 goto free_pages; 391 } 392 return dmabuf; 393 394 free_pages: 395 for_each_sgtable_sg(table, sg, i) { 396 struct page *p = sg_page(sg); 397 398 __free_pages(p, compound_order(p)); 399 } 400 sg_free_table(table); 401 free_buffer: 402 list_for_each_entry_safe(page, tmp_page, &pages, lru) 403 __free_pages(page, compound_order(page)); 404 kfree(buffer); 405 406 return ERR_PTR(ret); 407 } 408 409 static const struct dma_heap_ops system_heap_ops = { 410 .allocate = system_heap_allocate, 411 }; 412 413 static int __init system_heap_create(void) 414 { 415 struct dma_heap_export_info exp_info; 416 struct dma_heap *sys_heap; 417 418 exp_info.name = "system"; 419 exp_info.ops = &system_heap_ops; 420 exp_info.priv = NULL; 421 422 sys_heap = dma_heap_add(&exp_info); 423 if (IS_ERR(sys_heap)) 424 return PTR_ERR(sys_heap); 425 426 return 0; 427 } 428 module_init(system_heap_create); 429