1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/mman.h> 5 #include <linux/dma-mapping.h> 6 7 #include "ionic_fw.h" 8 #include "ionic_ibdev.h" 9 10 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va) 11 { 12 u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1; 13 u64 dma; 14 15 if (!buf->tbl_pages) 16 return cpu_to_le64(0); 17 18 if (buf->tbl_pages > 1) 19 return cpu_to_le64(buf->tbl_dma); 20 21 if (buf->tbl_buf) 22 dma = le64_to_cpu(buf->tbl_buf[0]); 23 else 24 dma = buf->tbl_dma; 25 26 return cpu_to_le64(dma + (va & pg_mask)); 27 } 28 29 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma) 30 { 31 if (unlikely(buf->tbl_pages == buf->tbl_limit)) 32 return -ENOMEM; 33 34 if (buf->tbl_buf) 35 buf->tbl_buf[buf->tbl_pages] = cpu_to_le64(dma); 36 else 37 buf->tbl_dma = dma; 38 39 ++buf->tbl_pages; 40 41 return 0; 42 } 43 44 static int ionic_tbl_buf_alloc(struct ionic_ibdev *dev, 45 struct ionic_tbl_buf *buf) 46 { 47 int rc; 48 49 buf->tbl_size = buf->tbl_limit * sizeof(*buf->tbl_buf); 50 buf->tbl_buf = kmalloc(buf->tbl_size, GFP_KERNEL); 51 if (!buf->tbl_buf) 52 return -ENOMEM; 53 54 buf->tbl_dma = dma_map_single(dev->lif_cfg.hwdev, buf->tbl_buf, 55 buf->tbl_size, DMA_TO_DEVICE); 56 rc = dma_mapping_error(dev->lif_cfg.hwdev, buf->tbl_dma); 57 if (rc) { 58 kfree(buf->tbl_buf); 59 return rc; 60 } 61 62 return 0; 63 } 64 65 static int ionic_pgtbl_umem(struct ionic_tbl_buf *buf, struct ib_umem *umem) 66 { 67 struct ib_block_iter biter; 68 u64 page_dma; 69 int rc; 70 71 rdma_umem_for_each_dma_block(umem, &biter, BIT_ULL(buf->page_size_log2)) { 72 page_dma = rdma_block_iter_dma_address(&biter); 73 rc = ionic_pgtbl_page(buf, page_dma); 74 if (rc) 75 return rc; 76 } 77 78 return 0; 79 } 80 81 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf) 82 { 83 if (buf->tbl_buf) 84 dma_unmap_single(dev->lif_cfg.hwdev, buf->tbl_dma, 85 buf->tbl_size, DMA_TO_DEVICE); 86 87 kfree(buf->tbl_buf); 88 memset(buf, 0, sizeof(*buf)); 89 } 90 91 int ionic_pgtbl_init(struct ionic_ibdev *dev, 92 struct ionic_tbl_buf *buf, 93 struct ib_umem *umem, 94 dma_addr_t dma, 95 int limit, 96 u64 page_size) 97 { 98 int rc; 99 100 memset(buf, 0, sizeof(*buf)); 101 102 if (umem) { 103 limit = ib_umem_num_dma_blocks(umem, page_size); 104 buf->page_size_log2 = order_base_2(page_size); 105 } 106 107 if (limit < 1) 108 return -EINVAL; 109 110 buf->tbl_limit = limit; 111 112 /* skip pgtbl if contiguous / direct translation */ 113 if (limit > 1) { 114 rc = ionic_tbl_buf_alloc(dev, buf); 115 if (rc) 116 return rc; 117 } 118 119 if (umem) 120 rc = ionic_pgtbl_umem(buf, umem); 121 else 122 rc = ionic_pgtbl_page(buf, dma); 123 124 if (rc) 125 goto err_unbuf; 126 127 return 0; 128 129 err_unbuf: 130 ionic_pgtbl_unbuf(dev, buf); 131 return rc; 132 } 133