1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2020 Intel Corporation. All rights reserved. 5 */ 6 7 #ifndef IB_UMEM_H 8 #define IB_UMEM_H 9 10 #include <linux/list.h> 11 #include <linux/scatterlist.h> 12 #include <linux/workqueue.h> 13 #include <rdma/ib_verbs.h> 14 15 struct ib_ucontext; 16 struct ib_umem_odp; 17 struct dma_buf_attach_ops; 18 19 struct ib_umem { 20 struct ib_device *ibdev; 21 struct mm_struct *owning_mm; 22 u64 iova; 23 size_t length; 24 unsigned long address; 25 u32 writable : 1; 26 u32 is_odp : 1; 27 u32 is_dmabuf : 1; 28 struct sg_append_table sgt_append; 29 }; 30 31 struct ib_umem_dmabuf { 32 struct ib_umem umem; 33 struct dma_buf_attachment *attach; 34 struct sg_table *sgt; 35 struct scatterlist *first_sg; 36 struct scatterlist *last_sg; 37 unsigned long first_sg_offset; 38 unsigned long last_sg_trim; 39 void *private; 40 u8 pinned : 1; 41 u8 revoked : 1; 42 }; 43 44 static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem) 45 { 46 return container_of(umem, struct ib_umem_dmabuf, umem); 47 } 48 49 /* Returns the offset of the umem start relative to the first page. */ 50 static inline int ib_umem_offset(struct ib_umem *umem) 51 { 52 return umem->address & ~PAGE_MASK; 53 } 54 55 static inline dma_addr_t ib_umem_start_dma_addr(struct ib_umem *umem) 56 { 57 return sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem); 58 } 59 60 static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem, 61 unsigned long pgsz) 62 { 63 return ib_umem_start_dma_addr(umem) & (pgsz - 1); 64 } 65 66 static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem, 67 unsigned long pgsz) 68 { 69 return (size_t)((ALIGN(umem->iova + umem->length, pgsz) - 70 ALIGN_DOWN(umem->iova, pgsz))) / 71 pgsz; 72 } 73 74 static inline size_t ib_umem_num_pages(struct ib_umem *umem) 75 { 76 return ib_umem_num_dma_blocks(umem, PAGE_SIZE); 77 } 78 #ifdef CONFIG_INFINIBAND_USER_MEM 79 80 struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, 81 size_t size, int access); 82 void ib_umem_release(struct ib_umem *umem); 83 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 84 size_t length); 85 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 86 unsigned long pgsz_bitmap, 87 unsigned long virt); 88 89 /** 90 * ib_umem_find_best_pgoff - Find best HW page size 91 * 92 * @umem: umem struct 93 * @pgsz_bitmap: bitmap of HW supported page sizes 94 * @pgoff_bitmask: Mask of bits that can be represented with an offset 95 * 96 * This is very similar to ib_umem_find_best_pgsz() except instead of accepting 97 * an IOVA it accepts a bitmask specifying what address bits can be represented 98 * with a page offset. 99 * 100 * For instance if the HW has multiple page sizes, requires 64 byte alignemnt, 101 * and can support aligned offsets up to 4032 then pgoff_bitmask would be 102 * "111111000000". 103 * 104 * If the pgoff_bitmask requires either alignment in the low bit or an 105 * unavailable page size for the high bits, this function returns 0. 106 * 107 * Returns: best HW page size for the parameters or 0 if none available 108 * for the given parameters. 109 */ 110 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 111 unsigned long pgsz_bitmap, 112 u64 pgoff_bitmask) 113 { 114 dma_addr_t dma_addr; 115 116 dma_addr = ib_umem_start_dma_addr(umem); 117 return ib_umem_find_best_pgsz(umem, pgsz_bitmap, 118 dma_addr & pgoff_bitmask); 119 } 120 121 static inline bool ib_umem_is_contiguous(struct ib_umem *umem) 122 { 123 dma_addr_t dma_addr; 124 unsigned long pgsz; 125 126 /* 127 * Select the smallest aligned page that can contain the whole umem if 128 * it was contiguous. 129 */ 130 dma_addr = ib_umem_start_dma_addr(umem); 131 pgsz = roundup_pow_of_two((dma_addr ^ (umem->length - 1 + dma_addr)) + 1); 132 return !!ib_umem_find_best_pgoff(umem, pgsz, U64_MAX); 133 } 134 135 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 136 unsigned long offset, size_t size, 137 int fd, int access, 138 const struct dma_buf_attach_ops *ops); 139 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 140 unsigned long offset, 141 size_t size, int fd, 142 int access); 143 struct ib_umem_dmabuf * 144 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 145 struct device *dma_device, 146 unsigned long offset, size_t size, 147 int fd, int access); 148 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf); 149 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf); 150 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf); 151 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); 152 153 #else /* CONFIG_INFINIBAND_USER_MEM */ 154 155 #include <linux/err.h> 156 157 static inline struct ib_umem *ib_umem_get(struct ib_device *device, 158 unsigned long addr, size_t size, 159 int access) 160 { 161 return ERR_PTR(-EOPNOTSUPP); 162 } 163 static inline void ib_umem_release(struct ib_umem *umem) { } 164 static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 165 size_t length) { 166 return -EOPNOTSUPP; 167 } 168 static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 169 unsigned long pgsz_bitmap, 170 unsigned long virt) 171 { 172 return 0; 173 } 174 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 175 unsigned long pgsz_bitmap, 176 u64 pgoff_bitmask) 177 { 178 return 0; 179 } 180 static inline 181 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 182 unsigned long offset, 183 size_t size, int fd, 184 int access, 185 struct dma_buf_attach_ops *ops) 186 { 187 return ERR_PTR(-EOPNOTSUPP); 188 } 189 static inline struct ib_umem_dmabuf * 190 ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset, 191 size_t size, int fd, int access) 192 { 193 return ERR_PTR(-EOPNOTSUPP); 194 } 195 196 static inline struct ib_umem_dmabuf * 197 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 198 struct device *dma_device, 199 unsigned long offset, size_t size, 200 int fd, int access) 201 { 202 return ERR_PTR(-EOPNOTSUPP); 203 } 204 205 static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 206 { 207 return -EOPNOTSUPP; 208 } 209 static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { } 210 static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { } 211 static inline void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) {} 212 213 #endif /* CONFIG_INFINIBAND_USER_MEM */ 214 #endif /* IB_UMEM_H */ 215