1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */ 3 4 #include <linux/mman.h> 5 #include <linux/dma-mapping.h> 6 7 #include "ionic_fw.h" 8 #include "ionic_ibdev.h" 9 10 __le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va) 11 { 12 u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1; 13 u64 dma; 14 15 if (!buf->tbl_pages) 16 return cpu_to_le64(0); 17 18 if (buf->tbl_pages > 1) 19 return cpu_to_le64(buf->tbl_dma); 20 21 if (buf->tbl_buf) 22 dma = le64_to_cpu(buf->tbl_buf[0]); 23 else 24 dma = buf->tbl_dma; 25 26 return cpu_to_le64(dma + (va & pg_mask)); 27 } 28 29 __be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va) 30 { 31 if (buf->tbl_pages > 1) { 32 u64 pg_mask = BIT_ULL(buf->page_size_log2) - 1; 33 34 return cpu_to_be64(va & pg_mask); 35 } 36 37 return 0; 38 } 39 40 int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma) 41 { 42 if (unlikely(buf->tbl_pages == buf->tbl_limit)) 43 return -ENOMEM; 44 45 if (buf->tbl_buf) 46 buf->tbl_buf[buf->tbl_pages] = cpu_to_le64(dma); 47 else 48 buf->tbl_dma = dma; 49 50 ++buf->tbl_pages; 51 52 return 0; 53 } 54 55 static int ionic_tbl_buf_alloc(struct ionic_ibdev *dev, 56 struct ionic_tbl_buf *buf) 57 { 58 int rc; 59 60 buf->tbl_size = buf->tbl_limit * sizeof(*buf->tbl_buf); 61 buf->tbl_buf = kmalloc(buf->tbl_size, GFP_KERNEL); 62 if (!buf->tbl_buf) 63 return -ENOMEM; 64 65 buf->tbl_dma = dma_map_single(dev->lif_cfg.hwdev, buf->tbl_buf, 66 buf->tbl_size, DMA_TO_DEVICE); 67 rc = dma_mapping_error(dev->lif_cfg.hwdev, buf->tbl_dma); 68 if (rc) { 69 kfree(buf->tbl_buf); 70 return rc; 71 } 72 73 return 0; 74 } 75 76 static int ionic_pgtbl_umem(struct ionic_tbl_buf *buf, struct ib_umem *umem) 77 { 78 struct ib_block_iter biter; 79 u64 page_dma; 80 int rc; 81 82 rdma_umem_for_each_dma_block(umem, &biter, BIT_ULL(buf->page_size_log2)) { 83 page_dma = rdma_block_iter_dma_address(&biter); 84 rc = ionic_pgtbl_page(buf, page_dma); 85 if (rc) 86 return rc; 87 } 88 89 return 0; 90 } 91 92 void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf) 93 { 94 if (buf->tbl_buf) 95 dma_unmap_single(dev->lif_cfg.hwdev, buf->tbl_dma, 96 buf->tbl_size, DMA_TO_DEVICE); 97 98 kfree(buf->tbl_buf); 99 memset(buf, 0, sizeof(*buf)); 100 } 101 102 int ionic_pgtbl_init(struct ionic_ibdev *dev, 103 struct ionic_tbl_buf *buf, 104 struct ib_umem *umem, 105 dma_addr_t dma, 106 int limit, 107 u64 page_size) 108 { 109 int rc; 110 111 memset(buf, 0, sizeof(*buf)); 112 113 if (umem) { 114 limit = ib_umem_num_dma_blocks(umem, page_size); 115 buf->page_size_log2 = order_base_2(page_size); 116 } 117 118 if (limit < 1) 119 return -EINVAL; 120 121 buf->tbl_limit = limit; 122 123 /* skip pgtbl if contiguous / direct translation */ 124 if (limit > 1) { 125 rc = ionic_tbl_buf_alloc(dev, buf); 126 if (rc) 127 return rc; 128 } 129 130 if (umem) 131 rc = ionic_pgtbl_umem(buf, umem); 132 else 133 rc = ionic_pgtbl_page(buf, dma); 134 135 if (rc) 136 goto err_unbuf; 137 138 return 0; 139 140 err_unbuf: 141 ionic_pgtbl_unbuf(dev, buf); 142 return rc; 143 } 144