1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2020 Intel Corporation. All rights reserved. 5 */ 6 7 #ifndef IB_UMEM_H 8 #define IB_UMEM_H 9 10 #include <linux/dma-direction.h> 11 #include <linux/scatterlist.h> 12 13 struct ib_device; 14 struct dma_buf_attach_ops; 15 16 struct ib_umem { 17 struct ib_device *ibdev; 18 struct mm_struct *owning_mm; 19 u64 iova; 20 size_t length; 21 unsigned long address; 22 unsigned long dma_attrs; 23 enum dma_data_direction dma_dir; 24 u32 writable : 1; 25 u32 is_odp : 1; 26 u32 is_dmabuf : 1; 27 struct sg_append_table sgt_append; 28 }; 29 30 struct ib_umem_dmabuf { 31 struct ib_umem umem; 32 struct dma_buf_attachment *attach; 33 struct sg_table *sgt; 34 struct scatterlist *first_sg; 35 struct scatterlist *last_sg; 36 unsigned long first_sg_offset; 37 unsigned long last_sg_trim; 38 void (*pinned_revoke)(void *priv); 39 void *private; 40 u8 pinned : 1; 41 u8 revoked : 1; 42 }; 43 44 static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem) 45 { 46 return container_of(umem, struct ib_umem_dmabuf, umem); 47 } 48 49 /* Returns the offset of the umem start relative to the first page. */ 50 static inline int ib_umem_offset(struct ib_umem *umem) 51 { 52 return umem->address & ~PAGE_MASK; 53 } 54 55 static inline dma_addr_t ib_umem_start_dma_addr(struct ib_umem *umem) 56 { 57 return sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem); 58 } 59 60 static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem, 61 unsigned long pgsz) 62 { 63 return ib_umem_start_dma_addr(umem) & (pgsz - 1); 64 } 65 66 static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem, 67 unsigned long pgsz) 68 { 69 return (size_t)((ALIGN(umem->iova + umem->length, pgsz) - 70 ALIGN_DOWN(umem->iova, pgsz))) / 71 pgsz; 72 } 73 74 static inline size_t ib_umem_num_pages(struct ib_umem *umem) 75 { 76 return ib_umem_num_dma_blocks(umem, PAGE_SIZE); 77 } 78 79 struct ib_udata; 80 struct ib_uverbs_buffer_desc; 81 struct uverbs_attr_bundle; 82 83 #ifdef CONFIG_INFINIBAND_USER_MEM 84 85 struct ib_umem *ib_umem_get_desc(struct ib_device *device, 86 const struct ib_uverbs_buffer_desc *desc, 87 int access); 88 struct ib_umem *ib_umem_get_attr(struct ib_device *device, 89 const struct uverbs_attr_bundle *attrs, 90 u16 attr_id, size_t size, int access); 91 struct ib_umem *ib_umem_get_attr_or_va(struct ib_device *device, 92 const struct uverbs_attr_bundle *attrs, 93 u16 attr_id, u64 addr, size_t size, 94 int access); 95 struct ib_umem *ib_umem_get_cq_buf(struct ib_device *device, 96 const struct uverbs_attr_bundle *attrs, 97 size_t size, int access); 98 struct ib_umem *ib_umem_get_cq_buf_or_va(struct ib_device *device, 99 const struct uverbs_attr_bundle *attrs, 100 u64 addr, size_t size, int access); 101 102 static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, 103 unsigned long addr, size_t size, 104 int access) 105 { 106 return ib_umem_get_attr_or_va(device, NULL, 0, addr, size, access); 107 } 108 109 void ib_umem_release(struct ib_umem *umem); 110 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 111 size_t length); 112 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 113 unsigned long pgsz_bitmap, 114 u64 virt); 115 116 /** 117 * ib_umem_find_best_pgoff - Find best HW page size 118 * 119 * @umem: umem struct 120 * @pgsz_bitmap: bitmap of HW supported page sizes 121 * @pgoff_bitmask: Mask of bits that can be represented with an offset 122 * 123 * This is very similar to ib_umem_find_best_pgsz() except instead of accepting 124 * an IOVA it accepts a bitmask specifying what address bits can be represented 125 * with a page offset. 126 * 127 * For instance if the HW has multiple page sizes, requires 64 byte alignemnt, 128 * and can support aligned offsets up to 4032 then pgoff_bitmask would be 129 * "111111000000". 130 * 131 * If the pgoff_bitmask requires either alignment in the low bit or an 132 * unavailable page size for the high bits, this function returns 0. 133 * 134 * Returns: best HW page size for the parameters or 0 if none available 135 * for the given parameters. 136 */ 137 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 138 unsigned long pgsz_bitmap, 139 u64 pgoff_bitmask) 140 { 141 dma_addr_t dma_addr; 142 143 dma_addr = ib_umem_start_dma_addr(umem); 144 return ib_umem_find_best_pgsz(umem, pgsz_bitmap, 145 dma_addr & pgoff_bitmask); 146 } 147 148 static inline bool ib_umem_is_contiguous(struct ib_umem *umem) 149 { 150 unsigned long pgsz; 151 152 pgsz = ib_umem_find_best_pgsz(umem, ULONG_MAX, 153 ib_umem_start_dma_addr(umem)); 154 return pgsz && ib_umem_num_dma_blocks(umem, pgsz) == 1; 155 } 156 157 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 158 unsigned long offset, size_t size, 159 int fd, int access, 160 const struct dma_buf_attach_ops *ops); 161 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 162 unsigned long offset, 163 size_t size, int fd, 164 int access); 165 struct ib_umem_dmabuf * 166 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 167 unsigned long offset, size_t size, 168 int fd, int access); 169 void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 170 void (*revoke)(void *priv), void *priv); 171 struct ib_umem_dmabuf * 172 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 173 struct device *dma_device, 174 unsigned long offset, size_t size, 175 int fd, int access); 176 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf); 177 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf); 178 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf); 179 void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf); 180 void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf); 181 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); 182 183 int ib_umem_check_rereg(struct ib_umem *umem, int flags, int new_access_flags); 184 185 #else /* CONFIG_INFINIBAND_USER_MEM */ 186 187 #include <linux/err.h> 188 189 static inline struct ib_umem * 190 ib_umem_get_desc(struct ib_device *device, 191 const struct ib_uverbs_buffer_desc *desc, int access) 192 { 193 return ERR_PTR(-EOPNOTSUPP); 194 } 195 static inline struct ib_umem *ib_umem_get_va(struct ib_device *device, 196 unsigned long addr, size_t size, 197 int access) 198 { 199 return ERR_PTR(-EOPNOTSUPP); 200 } 201 static inline struct ib_umem * 202 ib_umem_get_attr(struct ib_device *device, 203 const struct uverbs_attr_bundle *attrs, u16 attr_id, 204 size_t size, int access) 205 { 206 return ERR_PTR(-EOPNOTSUPP); 207 } 208 static inline struct ib_umem * 209 ib_umem_get_attr_or_va(struct ib_device *device, 210 const struct uverbs_attr_bundle *attrs, u16 attr_id, 211 u64 addr, size_t size, int access) 212 { 213 return ERR_PTR(-EOPNOTSUPP); 214 } 215 static inline struct ib_umem * 216 ib_umem_get_cq_buf(struct ib_device *device, 217 const struct uverbs_attr_bundle *attrs, size_t size, 218 int access) 219 { 220 return ERR_PTR(-EOPNOTSUPP); 221 } 222 static inline struct ib_umem * 223 ib_umem_get_cq_buf_or_va(struct ib_device *device, 224 const struct uverbs_attr_bundle *attrs, u64 addr, 225 size_t size, int access) 226 { 227 return ERR_PTR(-EOPNOTSUPP); 228 } 229 static inline void ib_umem_release(struct ib_umem *umem) { } 230 static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 231 size_t length) { 232 return -EOPNOTSUPP; 233 } 234 static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 235 unsigned long pgsz_bitmap, 236 u64 virt) 237 { 238 return 0; 239 } 240 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 241 unsigned long pgsz_bitmap, 242 u64 pgoff_bitmask) 243 { 244 return 0; 245 } 246 static inline bool ib_umem_is_contiguous(struct ib_umem *umem) 247 { 248 return false; 249 } 250 static inline 251 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 252 unsigned long offset, 253 size_t size, int fd, 254 int access, 255 const struct dma_buf_attach_ops *ops) 256 { 257 return ERR_PTR(-EOPNOTSUPP); 258 } 259 static inline struct ib_umem_dmabuf * 260 ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset, 261 size_t size, int fd, int access) 262 { 263 return ERR_PTR(-EOPNOTSUPP); 264 } 265 266 static inline struct ib_umem_dmabuf * 267 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 268 unsigned long offset, size_t size, 269 int fd, int access) 270 { 271 return ERR_PTR(-EOPNOTSUPP); 272 } 273 274 static inline void 275 ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 276 void (*revoke)(void *priv), void *priv) {} 277 278 static inline struct ib_umem_dmabuf * 279 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 280 struct device *dma_device, 281 unsigned long offset, size_t size, 282 int fd, int access) 283 { 284 return ERR_PTR(-EOPNOTSUPP); 285 } 286 287 static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 288 { 289 return -EOPNOTSUPP; 290 } 291 static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { } 292 static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { } 293 static inline void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf) {} 294 static inline void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf) {} 295 static inline void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) {} 296 297 static inline int ib_umem_check_rereg(struct ib_umem *umem, int flags, 298 int new_access_flags) 299 { 300 return -EOPNOTSUPP; 301 } 302 303 #endif /* CONFIG_INFINIBAND_USER_MEM */ 304 #endif /* IB_UMEM_H */ 305