1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2007 Cisco Systems. All rights reserved. 4 * Copyright (c) 2020 Intel Corporation. All rights reserved. 5 */ 6 7 #ifndef IB_UMEM_H 8 #define IB_UMEM_H 9 10 #include <linux/scatterlist.h> 11 12 struct ib_device; 13 struct dma_buf_attach_ops; 14 15 struct ib_umem { 16 struct ib_device *ibdev; 17 struct mm_struct *owning_mm; 18 u64 iova; 19 size_t length; 20 unsigned long address; 21 unsigned long dma_attrs; 22 u32 writable : 1; 23 u32 is_odp : 1; 24 u32 is_dmabuf : 1; 25 struct sg_append_table sgt_append; 26 }; 27 28 struct ib_umem_dmabuf { 29 struct ib_umem umem; 30 struct dma_buf_attachment *attach; 31 struct sg_table *sgt; 32 struct scatterlist *first_sg; 33 struct scatterlist *last_sg; 34 unsigned long first_sg_offset; 35 unsigned long last_sg_trim; 36 void (*pinned_revoke)(void *priv); 37 void *private; 38 u8 pinned : 1; 39 u8 revoked : 1; 40 }; 41 42 static inline struct ib_umem_dmabuf *to_ib_umem_dmabuf(struct ib_umem *umem) 43 { 44 return container_of(umem, struct ib_umem_dmabuf, umem); 45 } 46 47 /* Returns the offset of the umem start relative to the first page. */ 48 static inline int ib_umem_offset(struct ib_umem *umem) 49 { 50 return umem->address & ~PAGE_MASK; 51 } 52 53 static inline dma_addr_t ib_umem_start_dma_addr(struct ib_umem *umem) 54 { 55 return sg_dma_address(umem->sgt_append.sgt.sgl) + ib_umem_offset(umem); 56 } 57 58 static inline unsigned long ib_umem_dma_offset(struct ib_umem *umem, 59 unsigned long pgsz) 60 { 61 return ib_umem_start_dma_addr(umem) & (pgsz - 1); 62 } 63 64 static inline size_t ib_umem_num_dma_blocks(struct ib_umem *umem, 65 unsigned long pgsz) 66 { 67 return (size_t)((ALIGN(umem->iova + umem->length, pgsz) - 68 ALIGN_DOWN(umem->iova, pgsz))) / 69 pgsz; 70 } 71 72 static inline size_t ib_umem_num_pages(struct ib_umem *umem) 73 { 74 return ib_umem_num_dma_blocks(umem, PAGE_SIZE); 75 } 76 #ifdef CONFIG_INFINIBAND_USER_MEM 77 78 struct ib_umem *ib_umem_get(struct ib_device *device, unsigned long addr, 79 size_t size, int access); 80 void ib_umem_release(struct ib_umem *umem); 81 int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 82 size_t length); 83 unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 84 unsigned long pgsz_bitmap, 85 unsigned long virt); 86 87 /** 88 * ib_umem_find_best_pgoff - Find best HW page size 89 * 90 * @umem: umem struct 91 * @pgsz_bitmap: bitmap of HW supported page sizes 92 * @pgoff_bitmask: Mask of bits that can be represented with an offset 93 * 94 * This is very similar to ib_umem_find_best_pgsz() except instead of accepting 95 * an IOVA it accepts a bitmask specifying what address bits can be represented 96 * with a page offset. 97 * 98 * For instance if the HW has multiple page sizes, requires 64 byte alignemnt, 99 * and can support aligned offsets up to 4032 then pgoff_bitmask would be 100 * "111111000000". 101 * 102 * If the pgoff_bitmask requires either alignment in the low bit or an 103 * unavailable page size for the high bits, this function returns 0. 104 * 105 * Returns: best HW page size for the parameters or 0 if none available 106 * for the given parameters. 107 */ 108 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 109 unsigned long pgsz_bitmap, 110 u64 pgoff_bitmask) 111 { 112 dma_addr_t dma_addr; 113 114 dma_addr = ib_umem_start_dma_addr(umem); 115 return ib_umem_find_best_pgsz(umem, pgsz_bitmap, 116 dma_addr & pgoff_bitmask); 117 } 118 119 static inline bool ib_umem_is_contiguous(struct ib_umem *umem) 120 { 121 dma_addr_t dma_addr; 122 unsigned long pgsz; 123 124 /* 125 * Select the smallest aligned page that can contain the whole umem if 126 * it was contiguous. 127 */ 128 dma_addr = ib_umem_start_dma_addr(umem); 129 pgsz = roundup_pow_of_two((dma_addr ^ (umem->length - 1 + dma_addr)) + 1); 130 return !!ib_umem_find_best_pgoff(umem, pgsz, U64_MAX); 131 } 132 133 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 134 unsigned long offset, size_t size, 135 int fd, int access, 136 const struct dma_buf_attach_ops *ops); 137 struct ib_umem_dmabuf *ib_umem_dmabuf_get_pinned(struct ib_device *device, 138 unsigned long offset, 139 size_t size, int fd, 140 int access); 141 struct ib_umem_dmabuf * 142 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 143 unsigned long offset, size_t size, 144 int fd, int access); 145 void ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 146 void (*revoke)(void *priv), void *priv); 147 struct ib_umem_dmabuf * 148 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 149 struct device *dma_device, 150 unsigned long offset, size_t size, 151 int fd, int access); 152 int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf); 153 void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf); 154 void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf); 155 void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf); 156 void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf); 157 void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf); 158 159 #else /* CONFIG_INFINIBAND_USER_MEM */ 160 161 #include <linux/err.h> 162 163 static inline struct ib_umem *ib_umem_get(struct ib_device *device, 164 unsigned long addr, size_t size, 165 int access) 166 { 167 return ERR_PTR(-EOPNOTSUPP); 168 } 169 static inline void ib_umem_release(struct ib_umem *umem) { } 170 static inline int ib_umem_copy_from(void *dst, struct ib_umem *umem, size_t offset, 171 size_t length) { 172 return -EOPNOTSUPP; 173 } 174 static inline unsigned long ib_umem_find_best_pgsz(struct ib_umem *umem, 175 unsigned long pgsz_bitmap, 176 unsigned long virt) 177 { 178 return 0; 179 } 180 static inline unsigned long ib_umem_find_best_pgoff(struct ib_umem *umem, 181 unsigned long pgsz_bitmap, 182 u64 pgoff_bitmask) 183 { 184 return 0; 185 } 186 static inline 187 struct ib_umem_dmabuf *ib_umem_dmabuf_get(struct ib_device *device, 188 unsigned long offset, 189 size_t size, int fd, 190 int access, 191 struct dma_buf_attach_ops *ops) 192 { 193 return ERR_PTR(-EOPNOTSUPP); 194 } 195 static inline struct ib_umem_dmabuf * 196 ib_umem_dmabuf_get_pinned(struct ib_device *device, unsigned long offset, 197 size_t size, int fd, int access) 198 { 199 return ERR_PTR(-EOPNOTSUPP); 200 } 201 202 static inline struct ib_umem_dmabuf * 203 ib_umem_dmabuf_get_pinned_revocable_and_lock(struct ib_device *device, 204 unsigned long offset, size_t size, 205 int fd, int access) 206 { 207 return ERR_PTR(-EOPNOTSUPP); 208 } 209 210 static inline void 211 ib_umem_dmabuf_set_revoke_locked(struct ib_umem_dmabuf *umem_dmabuf, 212 void (*revoke)(void *priv), void *priv) {} 213 214 static inline struct ib_umem_dmabuf * 215 ib_umem_dmabuf_get_pinned_with_dma_device(struct ib_device *device, 216 struct device *dma_device, 217 unsigned long offset, size_t size, 218 int fd, int access) 219 { 220 return ERR_PTR(-EOPNOTSUPP); 221 } 222 223 static inline int ib_umem_dmabuf_map_pages(struct ib_umem_dmabuf *umem_dmabuf) 224 { 225 return -EOPNOTSUPP; 226 } 227 static inline void ib_umem_dmabuf_unmap_pages(struct ib_umem_dmabuf *umem_dmabuf) { } 228 static inline void ib_umem_dmabuf_release(struct ib_umem_dmabuf *umem_dmabuf) { } 229 static inline void ib_umem_dmabuf_revoke_lock(struct ib_umem_dmabuf *umem_dmabuf) {} 230 static inline void ib_umem_dmabuf_revoke_unlock(struct ib_umem_dmabuf *umem_dmabuf) {} 231 static inline void ib_umem_dmabuf_revoke(struct ib_umem_dmabuf *umem_dmabuf) {} 232 233 #endif /* CONFIG_INFINIBAND_USER_MEM */ 234 #endif /* IB_UMEM_H */ 235