1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2020 Intel Corporation. All rights reserved. 5 */ 6 7 #ifndef IB_UMEM_H 8 #define IB_UMEM_H 9 10 #include <linux/scatterlist.h> 11 12 struct ib_device; 13 struct dma_buf_attach_ops; 14 15 struct ib_umem { 16 struct ib_device *ibdev; 17 struct mm_struct *owning_mm; 18 u64 iova; 19 size_t length; 20 unsigned long address; 21 unsigned long dma_attrs; 22 u32 writable : 1; 23 u32 is_odp : 1; 24 u32 is_dmabuf : 1; 25 struct sg_append_table sgt_append; 26 }; 27 28 struct ib_umem_dmabuf { 29 struct ib_umem umem; 30 struct dma_buf_attachment *attach; 31 struct sg_table *sgt; 32 struct scatterlist *first_sg; 33 struct scatterlist *last_sg; 34 unsigned long first_sg_offset; 35 unsigned long last_sg_trim; 36 void (*pinned_revoke)(void *priv); 37 void *private; 38 u8 pinned : 1; 39 u8 revoked : 1; 40 }; 41 42 static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem) 43 { 44 return container_of(umem, struct ib_umem_dmabuf, umem); 45 } 46 47 /* Returns the offset of the umem start relative to the first page. */ 48 static inline int ib_umem_offset(struct ib_umem *umem) 49 { 50 return umem->address & ~PAGE_MASK; 51 } 52 53 static inline dma_addr_t ib_umem_start_dma_addr(struct ib_umem *umem) 54 { 55 return sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem); 56 } 57 58 static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem, 59 unsigned long pgsz) 60 { 61 return ib_umem_start_dma_addr(umem) & (pgsz - 1); 62 } 63 64 static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem, 65 unsigned long pgsz) 66 { 67 return (size_t)((ALIGN(umem->iova + umem->length, pgsz) - 68 ALIGN_DOWN(umem->iova, pgsz))) / 69 pgsz; 70 } 71 72 static inline size_t ib_umem_num_pages(struct ib_umem *umem) 73 { 74 return ib_umem_num_dma_blocks(umem, PAGE_SIZE); 75 } 76 77 struct ib_udata; 78 struct ib_uverbs_buffer_desc; 79 struct uverbs_attr_bundle; 80 81 #ifdef CONFIG_INFINIBAND_USER_MEM 82 83 struct ib_umem *ib_umem_get_desc(struct ib_device *device, 84 const struct ib_uverbs_buffer_desc *desc, 85 int access); 86 struct ib_umem *ib_umem_get_attr(struct ib_device *device, 87 const struct uverbs_attr_bundle *attrs, 88 u16 attr_id, size_t size, int access); 89 struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, 90 const struct uverbs_attr_bundle *attrs, 91 u16 attr_id, u64 addr, size_t size, 92 int access); 93 struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, 94 const struct uverbs_attr_bundle *attrs, 95 size_t size, int access); 96 struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, 97 const struct uverbs_attr_bundle *attrs, 98 u64 addr, size_t size, int access); 99 100 static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, 101 unsigned long addr, size_t size, 102 int access) 103 { 104 return ib_umem_get_attr_or_va(device, NULL, 0, addr, size, access); 105 } 106 107 void ib_umem_release(struct ib_umem *umem); 108 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 109 size_t length); 110 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 111 unsigned long pgsz_bitmap, 112 u64 virt); 113 114 /** 115 * ib_umem_find_best_pgoff - Find best HW page size 116 * 117 * @umem: umem struct 118 * @pgsz_bitmap: bitmap of HW supported page sizes 119 * @pgoff_bitmask: Mask of bits that can be represented with an offset 120 * 121 * This is very similar to ib_umem_find_best_pgsz() except instead of accepting 122 * an IOVA it accepts a bitmask specifying what address bits can be represented 123 * with a page offset. 124 * 125 * For instance if the HW has multiple page sizes, requires 64 byte alignemnt, 126 * and can support aligned offsets up to 4032 then pgoff_bitmask would be 127 * "111111000000". 128 * 129 * If the pgoff_bitmask requires either alignment in the low bit or an 130 * unavailable page size for the high bits, this function returns 0. 131 * 132 * Returns: best HW page size for the parameters or 0 if none available 133 * for the given parameters. 134 */ 135 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 136 unsigned long pgsz_bitmap, 137 u64 pgoff_bitmask) 138 { 139 dma_addr_t dma_addr; 140 141 dma_addr = ib_umem_start_dma_addr(umem); 142 return ib_umem_find_best_pgsz(umem, pgsz_bitmap, 143 dma_addr & pgoff_bitmask); 144 } 145 146 static inline bool ib_umem_is_contiguous(struct ib_umem *umem) 147 { 148 unsigned long pgsz; 149 150 pgsz = ib_umem_find_best_pgsz(umem, ULONG_MAX, 151 ib_umem_start_dma_addr(umem)); 152 return pgsz && ib_umem_num_dma_blocks(umem, pgsz) == 1; 153 } 154 155 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 156 unsigned long offset, size_t size, 157 int fd, int access, 158 const struct dma_buf_attach_ops *ops); 159 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 160 unsigned long offset, 161 size_t size, int fd, 162 int access); 163 struct ib_umem_dmabuf * 164 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 165 unsigned long offset, size_t size, 166 int fd, int access); 167 void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 168 void (*revoke)(void *priv), void *priv); 169 struct ib_umem_dmabuf * 170 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 171 struct device *dma_device, 172 unsigned long offset, size_t size, 173 int fd, int access); 174 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf); 175 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf); 176 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf); 177 void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf); 178 void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf); 179 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); 180 181 int ib_umem_check_rereg(struct ib_umem *umem, int flags, int new_access_flags); 182 183 #else /* CONFIG_INFINIBAND_USER_MEM */ 184 185 #include <linux/err.h> 186 187 static inline struct ib_umem * 188 ib_umem_get_desc(struct ib_device *device, 189 const struct ib_uverbs_buffer_desc *desc, int access) 190 { 191 return ERR_PTR(-EOPNOTSUPP); 192 } 193 static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, 194 unsigned long addr, size_t size, 195 int access) 196 { 197 return ERR_PTR(-EOPNOTSUPP); 198 } 199 static inline struct ib_umem * 200 ib_umem_get_attr(struct ib_device *device, 201 const struct uverbs_attr_bundle *attrs, u16 attr_id, 202 size_t size, int access) 203 { 204 return ERR_PTR(-EOPNOTSUPP); 205 } 206 static inline struct ib_umem * 207 ib_umem_get_attr_or_va(struct ib_device *device, 208 const struct uverbs_attr_bundle *attrs, u16 attr_id, 209 u64 addr, size_t size, int access) 210 { 211 return ERR_PTR(-EOPNOTSUPP); 212 } 213 static inline struct ib_umem * 214 ib_umem_get_cq_buf(struct ib_device *device, 215 const struct uverbs_attr_bundle *attrs, size_t size, 216 int access) 217 { 218 return ERR_PTR(-EOPNOTSUPP); 219 } 220 static inline struct ib_umem * 221 ib_umem_get_cq_buf_or_va(struct ib_device *device, 222 const struct uverbs_attr_bundle *attrs, u64 addr, 223 size_t size, int access) 224 { 225 return ERR_PTR(-EOPNOTSUPP); 226 } 227 static inline void ib_umem_release(struct ib_umem *umem) { } 228 static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 229 size_t length) { 230 return -EOPNOTSUPP; 231 } 232 static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 233 unsigned long pgsz_bitmap, 234 u64 virt) 235 { 236 return 0; 237 } 238 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 239 unsigned long pgsz_bitmap, 240 u64 pgoff_bitmask) 241 { 242 return 0; 243 } 244 static inline bool ib_umem_is_contiguous(struct ib_umem *umem) 245 { 246 return false; 247 } 248 static inline 249 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 250 unsigned long offset, 251 size_t size, int fd, 252 int access, 253 struct dma_buf_attach_ops *ops) 254 { 255 return ERR_PTR(-EOPNOTSUPP); 256 } 257 static inline struct ib_umem_dmabuf * 258 ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset, 259 size_t size, int fd, int access) 260 { 261 return ERR_PTR(-EOPNOTSUPP); 262 } 263 264 static inline struct ib_umem_dmabuf * 265 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 266 unsigned long offset, size_t size, 267 int fd, int access) 268 { 269 return ERR_PTR(-EOPNOTSUPP); 270 } 271 272 static inline void 273 ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 274 void (*revoke)(void *priv), void *priv) {} 275 276 static inline struct ib_umem_dmabuf * 277 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 278 struct device *dma_device, 279 unsigned long offset, size_t size, 280 int fd, int access) 281 { 282 return ERR_PTR(-EOPNOTSUPP); 283 } 284 285 static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 286 { 287 return -EOPNOTSUPP; 288 } 289 static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { } 290 static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { } 291 static inline void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf) {} 292 static inline void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf) {} 293 static inline void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) {} 294 295 static inline int ib_umem_check_rereg(struct ib_umem *umem, int flags, 296 int new_access_flags) 297 { 298 return -EOPNOTSUPP; 299 } 300 301 #endif /* CONFIG_INFINIBAND_USER_MEM */ 302 #endif /* IB_UMEM_H */ 303