1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2018-2020 Christoph Hellwig. 4 * 5 * DMA operations that map physical memory directly without using an IOMMU. 6 */ 7 #include <linux/memblock.h> /* for max_pfn */ 8 #include <linux/export.h> 9 #include <linux/mm.h> 10 #include <linux/dma-map-ops.h> 11 #include <linux/scatterlist.h> 12 #include <linux/pfn.h> 13 #include <linux/vmalloc.h> 14 #include <linux/set_memory.h> 15 #include <linux/slab.h> 16 #include <linux/pci-p2pdma.h> 17 #include "direct.h" 18 19 /* 20 * Most architectures use ZONE_DMA for the first 16 Megabytes, but some use 21 * it for entirely different regions. In that case the arch code needs to 22 * override the variable below for dma-direct to work properly. 23 */ 24 u64 zone_dma_limit __ro_after_init = DMA_BIT_MASK(24); 25 26 static inline dma_addr_t phys_to_dma_direct(struct device *dev, 27 phys_addr_t phys) 28 { 29 if (force_dma_unencrypted(dev)) 30 return phys_to_dma_unencrypted(dev, phys); 31 return phys_to_dma(dev, phys); 32 } 33 34 static inline struct page *dma_direct_to_page(struct device *dev, 35 dma_addr_t dma_addr) 36 { 37 return pfn_to_page(PHYS_PFN(dma_to_phys(dev, dma_addr))); 38 } 39 40 u64 dma_direct_get_required_mask(struct device *dev) 41 { 42 phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT; 43 u64 max_dma = phys_to_dma_direct(dev, phys); 44 45 return (1ULL << (fls64(max_dma) - 1)) * 2 - 1; 46 } 47 48 static gfp_t dma_direct_optimal_gfp_mask(struct device *dev, u64 *phys_limit) 49 { 50 u64 dma_limit = min_not_zero( 51 dev->coherent_dma_mask, 52 dev->bus_dma_limit); 53 54 /* 55 * Optimistically try the zone that the physical address mask falls 56 * into first. If that returns memory that isn't actually addressable 57 * we will fallback to the next lower zone and try again. 58 * 59 * Note that GFP_DMA32 and GFP_DMA are no ops without the corresponding 60 * zones. 61 */ 62 *phys_limit = dma_to_phys(dev, dma_limit); 63 if (*phys_limit <= zone_dma_limit) 64 return GFP_DMA; 65 if (*phys_limit <= DMA_BIT_MASK(32)) 66 return GFP_DMA32; 67 return 0; 68 } 69 70 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size) 71 { 72 dma_addr_t dma_addr = phys_to_dma_direct(dev, phys); 73 74 if (dma_addr == DMA_MAPPING_ERROR) 75 return false; 76 return dma_addr + size - 1 <= 77 min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); 78 } 79 80 static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size) 81 { 82 if (!force_dma_unencrypted(dev)) 83 return 0; 84 return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size)); 85 } 86 87 static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size) 88 { 89 int ret; 90 91 if (!force_dma_unencrypted(dev)) 92 return 0; 93 ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size)); 94 if (ret) 95 pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n"); 96 return ret; 97 } 98 99 static void __dma_direct_free_pages(struct device *dev, struct page *page, 100 size_t size) 101 { 102 if (swiotlb_free(dev, page, size)) 103 return; 104 dma_free_contiguous(dev, page, size); 105 } 106 107 static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size) 108 { 109 struct page *page = swiotlb_alloc(dev, size); 110 111 if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) { 112 swiotlb_free(dev, page, size); 113 return NULL; 114 } 115 116 return page; 117 } 118 119 static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size, 120 gfp_t gfp, bool allow_highmem) 121 { 122 int node = dev_to_node(dev); 123 struct page *page; 124 u64 phys_limit; 125 126 WARN_ON_ONCE(!PAGE_ALIGNED(size)); 127 128 if (is_swiotlb_for_alloc(dev)) 129 return dma_direct_alloc_swiotlb(dev, size); 130 131 gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); 132 page = dma_alloc_contiguous(dev, size, gfp); 133 if (page) { 134 if (dma_coherent_ok(dev, page_to_phys(page), size) && 135 (allow_highmem || !PageHighMem(page))) 136 return page; 137 138 dma_free_contiguous(dev, page, size); 139 } 140 141 while ((page = alloc_pages_node(node, gfp, get_order(size))) 142 && !dma_coherent_ok(dev, page_to_phys(page), size)) { 143 __free_pages(page, get_order(size)); 144 145 if (IS_ENABLED(CONFIG_ZONE_DMA32) && 146 phys_limit < DMA_BIT_MASK(64) && 147 !(gfp & (GFP_DMA32 | GFP_DMA))) 148 gfp |= GFP_DMA32; 149 else if (IS_ENABLED(CONFIG_ZONE_DMA) && !(gfp & GFP_DMA)) 150 gfp = (gfp & ~GFP_DMA32) | GFP_DMA; 151 else 152 return NULL; 153 } 154 155 return page; 156 } 157 158 /* 159 * Check if a potentially blocking operations needs to dip into the atomic 160 * pools for the given device/gfp. 161 */ 162 static bool dma_direct_use_pool(struct device *dev, gfp_t gfp) 163 { 164 return !gfpflags_allow_blocking(gfp) && !is_swiotlb_for_alloc(dev); 165 } 166 167 static void *dma_direct_alloc_from_pool(struct device *dev, size_t size, 168 dma_addr_t *dma_handle, gfp_t gfp) 169 { 170 struct page *page; 171 u64 phys_limit; 172 void *ret; 173 174 if (WARN_ON_ONCE(!IS_ENABLED(CONFIG_DMA_COHERENT_POOL))) 175 return NULL; 176 177 gfp |= dma_direct_optimal_gfp_mask(dev, &phys_limit); 178 page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok); 179 if (!page) 180 return NULL; 181 *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); 182 return ret; 183 } 184 185 static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size, 186 dma_addr_t *dma_handle, gfp_t gfp) 187 { 188 struct page *page; 189 190 page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); 191 if (!page) 192 return NULL; 193 194 /* remove any dirty cache lines on the kernel alias */ 195 if (!PageHighMem(page)) 196 arch_dma_prep_coherent(page, size); 197 198 /* return the page pointer as the opaque cookie */ 199 *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); 200 return page; 201 } 202 203 void *dma_direct_alloc(struct device *dev, size_t size, 204 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 205 { 206 bool remap = false, set_uncached = false; 207 struct page *page; 208 void *ret; 209 210 size = PAGE_ALIGN(size); 211 if (attrs & DMA_ATTR_NO_WARN) 212 gfp |= __GFP_NOWARN; 213 214 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && 215 !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) 216 return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp); 217 218 if (!dev_is_dma_coherent(dev)) { 219 if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && 220 !is_swiotlb_for_alloc(dev)) 221 return arch_dma_alloc(dev, size, dma_handle, gfp, 222 attrs); 223 224 /* 225 * If there is a global pool, always allocate from it for 226 * non-coherent devices. 227 */ 228 if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL)) 229 return dma_alloc_from_global_coherent(dev, size, 230 dma_handle); 231 232 /* 233 * Otherwise we require the architecture to either be able to 234 * mark arbitrary parts of the kernel direct mapping uncached, 235 * or remapped it uncached. 236 */ 237 set_uncached = IS_ENABLED(CONFIG_ARCH_HAS_DMA_SET_UNCACHED); 238 remap = IS_ENABLED(CONFIG_DMA_DIRECT_REMAP); 239 if (!set_uncached && !remap) { 240 pr_warn_once("coherent DMA allocations not supported on this platform.\n"); 241 return NULL; 242 } 243 } 244 245 /* 246 * Remapping or decrypting memory may block, allocate the memory from 247 * the atomic pools instead if we aren't allowed block. 248 */ 249 if ((remap || force_dma_unencrypted(dev)) && 250 dma_direct_use_pool(dev, gfp)) 251 return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); 252 253 /* we always manually zero the memory once we are done */ 254 page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true); 255 if (!page) 256 return NULL; 257 258 /* 259 * dma_alloc_contiguous can return highmem pages depending on a 260 * combination the cma= arguments and per-arch setup. These need to be 261 * remapped to return a kernel virtual address. 262 */ 263 if (PageHighMem(page)) { 264 remap = true; 265 set_uncached = false; 266 } 267 268 if (remap) { 269 pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs); 270 271 if (force_dma_unencrypted(dev)) 272 prot = pgprot_decrypted(prot); 273 274 /* remove any dirty cache lines on the kernel alias */ 275 arch_dma_prep_coherent(page, size); 276 277 /* create a coherent mapping */ 278 ret = dma_common_contiguous_remap(page, size, prot, 279 __builtin_return_address(0)); 280 if (!ret) 281 goto out_free_pages; 282 } else { 283 ret = page_address(page); 284 if (dma_set_decrypted(dev, ret, size)) 285 goto out_leak_pages; 286 } 287 288 memset(ret, 0, size); 289 290 if (set_uncached) { 291 arch_dma_prep_coherent(page, size); 292 ret = arch_dma_set_uncached(ret, size); 293 if (IS_ERR(ret)) 294 goto out_encrypt_pages; 295 } 296 297 *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); 298 return ret; 299 300 out_encrypt_pages: 301 if (dma_set_encrypted(dev, page_address(page), size)) 302 return NULL; 303 out_free_pages: 304 __dma_direct_free_pages(dev, page, size); 305 return NULL; 306 out_leak_pages: 307 return NULL; 308 } 309 310 void dma_direct_free(struct device *dev, size_t size, 311 void *cpu_addr, dma_addr_t dma_addr, unsigned long attrs) 312 { 313 unsigned int page_order = get_order(size); 314 315 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) && 316 !force_dma_unencrypted(dev) && !is_swiotlb_for_alloc(dev)) { 317 /* cpu_addr is a struct page cookie, not a kernel address */ 318 dma_free_contiguous(dev, cpu_addr, size); 319 return; 320 } 321 322 if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_ALLOC) && 323 !dev_is_dma_coherent(dev) && 324 !is_swiotlb_for_alloc(dev)) { 325 arch_dma_free(dev, size, cpu_addr, dma_addr, attrs); 326 return; 327 } 328 329 if (IS_ENABLED(CONFIG_DMA_GLOBAL_POOL) && 330 !dev_is_dma_coherent(dev)) { 331 if (!dma_release_from_global_coherent(page_order, cpu_addr)) 332 WARN_ON_ONCE(1); 333 return; 334 } 335 336 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ 337 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && 338 dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size))) 339 return; 340 341 if (is_vmalloc_addr(cpu_addr)) { 342 vunmap(cpu_addr); 343 } else { 344 if (IS_ENABLED(CONFIG_ARCH_HAS_DMA_CLEAR_UNCACHED)) 345 arch_dma_clear_uncached(cpu_addr, size); 346 if (dma_set_encrypted(dev, cpu_addr, size)) 347 return; 348 } 349 350 __dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size); 351 } 352 353 struct page *dma_direct_alloc_pages(struct device *dev, size_t size, 354 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 355 { 356 struct page *page; 357 void *ret; 358 359 if (force_dma_unencrypted(dev) && dma_direct_use_pool(dev, gfp)) 360 return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp); 361 362 page = __dma_direct_alloc_pages(dev, size, gfp, false); 363 if (!page) 364 return NULL; 365 366 ret = page_address(page); 367 if (dma_set_decrypted(dev, ret, size)) 368 goto out_leak_pages; 369 memset(ret, 0, size); 370 *dma_handle = phys_to_dma_direct(dev, page_to_phys(page)); 371 return page; 372 out_leak_pages: 373 return NULL; 374 } 375 376 void dma_direct_free_pages(struct device *dev, size_t size, 377 struct page *page, dma_addr_t dma_addr, 378 enum dma_data_direction dir) 379 { 380 void *vaddr = page_address(page); 381 382 /* If cpu_addr is not from an atomic pool, dma_free_from_pool() fails */ 383 if (IS_ENABLED(CONFIG_DMA_COHERENT_POOL) && 384 dma_free_from_pool(dev, vaddr, size)) 385 return; 386 387 if (dma_set_encrypted(dev, vaddr, size)) 388 return; 389 __dma_direct_free_pages(dev, page, size); 390 } 391 392 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ 393 defined(CONFIG_SWIOTLB) 394 void dma_direct_sync_sg_for_device(struct device *dev, 395 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 396 { 397 struct scatterlist *sg; 398 int i; 399 400 for_each_sg(sgl, sg, nents, i) { 401 phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); 402 403 swiotlb_sync_single_for_device(dev, paddr, sg->length, dir); 404 405 if (!dev_is_dma_coherent(dev)) 406 arch_sync_dma_for_device(paddr, sg->length, 407 dir); 408 } 409 } 410 #endif 411 412 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 413 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) || \ 414 defined(CONFIG_SWIOTLB) 415 void dma_direct_sync_sg_for_cpu(struct device *dev, 416 struct scatterlist *sgl, int nents, enum dma_data_direction dir) 417 { 418 struct scatterlist *sg; 419 int i; 420 421 for_each_sg(sgl, sg, nents, i) { 422 phys_addr_t paddr = dma_to_phys(dev, sg_dma_address(sg)); 423 424 if (!dev_is_dma_coherent(dev)) 425 arch_sync_dma_for_cpu(paddr, sg->length, dir); 426 427 swiotlb_sync_single_for_cpu(dev, paddr, sg->length, dir); 428 429 if (dir == DMA_FROM_DEVICE) 430 arch_dma_mark_clean(paddr, sg->length); 431 } 432 433 if (!dev_is_dma_coherent(dev)) 434 arch_sync_dma_for_cpu_all(); 435 } 436 437 /* 438 * Unmaps segments, except for ones marked as pci_p2pdma which do not 439 * require any further action as they contain a bus address. 440 */ 441 void dma_direct_unmap_sg(struct device *dev, struct scatterlist *sgl, 442 int nents, enum dma_data_direction dir, unsigned long attrs) 443 { 444 struct scatterlist *sg; 445 int i; 446 447 for_each_sg(sgl, sg, nents, i) { 448 if (sg_dma_is_bus_address(sg)) 449 sg_dma_unmark_bus_address(sg); 450 else 451 dma_direct_unmap_phys(dev, sg->dma_address, 452 sg_dma_len(sg), dir, attrs); 453 } 454 } 455 #endif 456 457 int dma_direct_map_sg(struct device *dev, struct scatterlist *sgl, int nents, 458 enum dma_data_direction dir, unsigned long attrs) 459 { 460 struct pci_p2pdma_map_state p2pdma_state = {}; 461 struct scatterlist *sg; 462 int i, ret; 463 464 for_each_sg(sgl, sg, nents, i) { 465 switch (pci_p2pdma_state(&p2pdma_state, dev, sg_page(sg))) { 466 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE: 467 /* 468 * Any P2P mapping that traverses the PCI host bridge 469 * must be mapped with CPU physical address and not PCI 470 * bus addresses. 471 */ 472 break; 473 case PCI_P2PDMA_MAP_NONE: 474 sg->dma_address = dma_direct_map_phys(dev, sg_phys(sg), 475 sg->length, dir, attrs); 476 if (sg->dma_address == DMA_MAPPING_ERROR) { 477 ret = -EIO; 478 goto out_unmap; 479 } 480 break; 481 case PCI_P2PDMA_MAP_BUS_ADDR: 482 sg->dma_address = pci_p2pdma_bus_addr_map(&p2pdma_state, 483 sg_phys(sg)); 484 sg_dma_mark_bus_address(sg); 485 continue; 486 default: 487 ret = -EREMOTEIO; 488 goto out_unmap; 489 } 490 sg_dma_len(sg) = sg->length; 491 } 492 493 return nents; 494 495 out_unmap: 496 dma_direct_unmap_sg(dev, sgl, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC); 497 return ret; 498 } 499 500 int dma_direct_get_sgtable(struct device *dev, struct sg_table *sgt, 501 void *cpu_addr, dma_addr_t dma_addr, size_t size, 502 unsigned long attrs) 503 { 504 struct page *page = dma_direct_to_page(dev, dma_addr); 505 int ret; 506 507 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 508 if (!ret) 509 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 510 return ret; 511 } 512 513 bool dma_direct_can_mmap(struct device *dev) 514 { 515 return dev_is_dma_coherent(dev) || 516 IS_ENABLED(CONFIG_DMA_NONCOHERENT_MMAP); 517 } 518 519 int dma_direct_mmap(struct device *dev, struct vm_area_struct *vma, 520 void *cpu_addr, dma_addr_t dma_addr, size_t size, 521 unsigned long attrs) 522 { 523 unsigned long user_count = vma_pages(vma); 524 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 525 unsigned long pfn = PHYS_PFN(dma_to_phys(dev, dma_addr)); 526 int ret = -ENXIO; 527 528 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs); 529 if (force_dma_unencrypted(dev)) 530 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot); 531 532 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret)) 533 return ret; 534 if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret)) 535 return ret; 536 537 if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff) 538 return -ENXIO; 539 return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff, 540 user_count << PAGE_SHIFT, vma->vm_page_prot); 541 } 542 543 int dma_direct_supported(struct device *dev, u64 mask) 544 { 545 u64 min_mask = (max_pfn - 1) << PAGE_SHIFT; 546 547 /* 548 * Because 32-bit DMA masks are so common we expect every architecture 549 * to be able to satisfy them - either by not supporting more physical 550 * memory, or by providing a ZONE_DMA32. If neither is the case, the 551 * architecture needs to use an IOMMU instead of the direct mapping. 552 */ 553 if (mask >= DMA_BIT_MASK(32)) 554 return 1; 555 556 /* 557 * This check needs to be against the actual bit mask value, so use 558 * phys_to_dma_unencrypted() here so that the SME encryption mask isn't 559 * part of the check. 560 */ 561 if (IS_ENABLED(CONFIG_ZONE_DMA)) 562 min_mask = min_t(u64, min_mask, zone_dma_limit); 563 return mask >= phys_to_dma_unencrypted(dev, min_mask); 564 } 565 566 static const struct bus_dma_region *dma_find_range(struct device *dev, 567 unsigned long start_pfn) 568 { 569 const struct bus_dma_region *m; 570 571 for (m = dev->dma_range_map; PFN_DOWN(m->size); m++) { 572 unsigned long cpu_start_pfn = PFN_DOWN(m->cpu_start); 573 574 if (start_pfn >= cpu_start_pfn && 575 start_pfn - cpu_start_pfn < PFN_DOWN(m->size)) 576 return m; 577 } 578 579 return NULL; 580 } 581 582 /* 583 * To check whether all ram resource ranges are covered by dma range map 584 * Returns 0 when further check is needed 585 * Returns 1 if there is some RAM range can't be covered by dma_range_map 586 */ 587 static int check_ram_in_range_map(unsigned long start_pfn, 588 unsigned long nr_pages, void *data) 589 { 590 unsigned long end_pfn = start_pfn + nr_pages; 591 struct device *dev = data; 592 593 while (start_pfn < end_pfn) { 594 const struct bus_dma_region *bdr; 595 596 bdr = dma_find_range(dev, start_pfn); 597 if (!bdr) 598 return 1; 599 600 start_pfn = PFN_DOWN(bdr->cpu_start) + PFN_DOWN(bdr->size); 601 } 602 603 return 0; 604 } 605 606 bool dma_direct_all_ram_mapped(struct device *dev) 607 { 608 if (!dev->dma_range_map) 609 return true; 610 return !walk_system_ram_range(0, PFN_DOWN(ULONG_MAX) + 1, dev, 611 check_ram_in_range_map); 612 } 613 614 size_t dma_direct_max_mapping_size(struct device *dev) 615 { 616 /* If SWIOTLB is active, use its maximum mapping size */ 617 if (is_swiotlb_active(dev) && 618 (dma_addressing_limited(dev) || is_swiotlb_force_bounce(dev))) 619 return swiotlb_max_mapping_size(dev); 620 return SIZE_MAX; 621 } 622 623 bool dma_direct_need_sync(struct device *dev, dma_addr_t dma_addr) 624 { 625 return !dev_is_dma_coherent(dev) || 626 swiotlb_find_pool(dev, dma_to_phys(dev, dma_addr)); 627 } 628 629 /** 630 * dma_direct_set_offset - Assign scalar offset for a single DMA range. 631 * @dev: device pointer; needed to "own" the alloced memory. 632 * @cpu_start: beginning of memory region covered by this offset. 633 * @dma_start: beginning of DMA/PCI region covered by this offset. 634 * @size: size of the region. 635 * 636 * This is for the simple case of a uniform offset which cannot 637 * be discovered by "dma-ranges". 638 * 639 * It returns -ENOMEM if out of memory, -EINVAL if a map 640 * already exists, 0 otherwise. 641 * 642 * Note: any call to this from a driver is a bug. The mapping needs 643 * to be described by the device tree or other firmware interfaces. 644 */ 645 int dma_direct_set_offset(struct device *dev, phys_addr_t cpu_start, 646 dma_addr_t dma_start, u64 size) 647 { 648 struct bus_dma_region *map; 649 u64 offset = (u64)cpu_start - (u64)dma_start; 650 651 if (dev->dma_range_map) { 652 dev_err(dev, "attempt to add DMA range to existing map\n"); 653 return -EINVAL; 654 } 655 656 if (!offset) 657 return 0; 658 659 map = kcalloc(2, sizeof(*map), GFP_KERNEL); 660 if (!map) 661 return -ENOMEM; 662 map[0].cpu_start = cpu_start; 663 map[0].dma_start = dma_start; 664 map[0].size = size; 665 dev->dma_range_map = map; 666 return 0; 667 } 668