1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #ifndef RXE_QUEUE_H 8 #define RXE_QUEUE_H 9 10 /* Implements a simple circular buffer that is shared between user 11 * and the driver and can be resized. The requested element size is 12 * rounded up to a power of 2 and the number of elements in the buffer 13 * is also rounded up to a power of 2. Since the queue is empty when 14 * the producer and consumer indices match the maximum capacity of the 15 * queue is one less than the number of element slots. 16 * 17 * Notes: 18 * - The driver indices are always masked off to q->index_mask 19 * before storing so do not need to be checked on reads. 20 * - The user whether user space or kernel is generally 21 * not trusted so its parameters are masked to make sure 22 * they do not access the queue out of bounds on reads. 23 * - The driver indices for queues must not be written 24 * by user so a local copy is used and a shared copy is 25 * stored when the local copy is changed. 26 * - By passing the type in the parameter list separate from q 27 * the compiler can eliminate the switch statement when the 28 * actual queue type is known when the function is called at 29 * compile time. 30 * - These queues are lock free. The user and driver must protect 31 * changes to their end of the queues with locks if more than one 32 * CPU can be accessing it at the same time. 33 */ 34 35 /** 36 * enum queue_type - type of queue 37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and 38 * read by client which may be a user space 39 * application or a kernel ulp. 40 * Used by rxe internals only. 41 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and 42 * read by rxe driver. 43 * Used by rxe internals only. 44 * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and 45 * read by rxe driver. 46 * Used by kernel verbs APIs only on 47 * behalf of ulps. 48 * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and 49 * read by kernel ulp. 50 * Used by kernel verbs APIs only on 51 * behalf of ulps. 52 */ 53 enum queue_type { 54 QUEUE_TYPE_TO_CLIENT, 55 QUEUE_TYPE_FROM_CLIENT, 56 QUEUE_TYPE_FROM_ULP, 57 QUEUE_TYPE_TO_ULP, 58 }; 59 60 struct rxe_queue_buf; 61 62 struct rxe_queue { 63 struct rxe_dev *rxe; 64 struct rxe_queue_buf *buf; 65 struct rxe_mmap_info *ip; 66 size_t buf_size; 67 size_t elem_size; 68 unsigned int log2_elem_size; 69 u32 index_mask; 70 enum queue_type type; 71 /* private copy of index for shared queues between 72 * driver and clients. Driver reads and writes 73 * this copy and then replicates to rxe_queue_buf 74 * for read access by clients. 75 */ 76 u32 index; 77 }; 78 79 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf, 80 struct ib_udata *udata, struct rxe_queue_buf *buf, 81 size_t buf_size, struct rxe_mmap_info **ip_p); 82 83 void rxe_queue_reset(struct rxe_queue *q); 84 85 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem, 86 unsigned int elem_size, enum queue_type type); 87 88 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, 89 unsigned int elem_size, struct ib_udata *udata, 90 struct mminfo __user *outbuf, 91 spinlock_t *producer_lock, spinlock_t *consumer_lock); 92 93 void rxe_queue_cleanup(struct rxe_queue *queue); 94 95 static inline u32 queue_next_index(struct rxe_queue *q, int index) 96 { 97 return (index + 1) & q->index_mask; 98 } 99 100 static inline u32 queue_get_producer(const struct rxe_queue *q, 101 enum queue_type type) 102 { 103 u32 prod; 104 105 switch (type) { 106 case QUEUE_TYPE_FROM_CLIENT: 107 /* used by rxe, client owns the index */ 108 prod = smp_load_acquire(&q->buf->producer_index); 109 break; 110 case QUEUE_TYPE_TO_CLIENT: 111 /* used by rxe which owns the index */ 112 prod = q->index; 113 break; 114 case QUEUE_TYPE_FROM_ULP: 115 /* used by ulp which owns the index */ 116 prod = q->buf->producer_index; 117 break; 118 case QUEUE_TYPE_TO_ULP: 119 /* used by ulp, rxe owns the index */ 120 prod = smp_load_acquire(&q->buf->producer_index); 121 break; 122 } 123 124 return prod; 125 } 126 127 static inline u32 queue_get_consumer(const struct rxe_queue *q, 128 enum queue_type type) 129 { 130 u32 cons; 131 132 switch (type) { 133 case QUEUE_TYPE_FROM_CLIENT: 134 /* used by rxe which owns the index */ 135 cons = q->index; 136 break; 137 case QUEUE_TYPE_TO_CLIENT: 138 /* used by rxe, client owns the index */ 139 cons = smp_load_acquire(&q->buf->consumer_index); 140 break; 141 case QUEUE_TYPE_FROM_ULP: 142 /* used by ulp, rxe owns the index */ 143 cons = smp_load_acquire(&q->buf->consumer_index); 144 break; 145 case QUEUE_TYPE_TO_ULP: 146 /* used by ulp which owns the index */ 147 cons = q->buf->consumer_index; 148 break; 149 } 150 151 return cons; 152 } 153 154 static inline int queue_empty(struct rxe_queue *q, enum queue_type type) 155 { 156 u32 prod = queue_get_producer(q, type); 157 u32 cons = queue_get_consumer(q, type); 158 159 return ((prod - cons) & q->index_mask) == 0; 160 } 161 162 static inline int queue_full(struct rxe_queue *q, enum queue_type type) 163 { 164 u32 prod = queue_get_producer(q, type); 165 u32 cons = queue_get_consumer(q, type); 166 167 return ((prod + 1 - cons) & q->index_mask) == 0; 168 } 169 170 static inline u32 queue_count(const struct rxe_queue *q, 171 enum queue_type type) 172 { 173 u32 prod = queue_get_producer(q, type); 174 u32 cons = queue_get_consumer(q, type); 175 176 return (prod - cons) & q->index_mask; 177 } 178 179 static inline void queue_advance_producer(struct rxe_queue *q, 180 enum queue_type type) 181 { 182 u32 prod; 183 184 switch (type) { 185 case QUEUE_TYPE_FROM_CLIENT: 186 /* used by rxe, client owns the index */ 187 if (WARN_ON(1)) 188 pr_warn("%s: attempt to advance client index\n", 189 __func__); 190 break; 191 case QUEUE_TYPE_TO_CLIENT: 192 /* used by rxe which owns the index */ 193 prod = q->index; 194 prod = (prod + 1) & q->index_mask; 195 q->index = prod; 196 /* release so client can read it safely */ 197 smp_store_release(&q->buf->producer_index, prod); 198 break; 199 case QUEUE_TYPE_FROM_ULP: 200 /* used by ulp which owns the index */ 201 prod = q->buf->producer_index; 202 prod = (prod + 1) & q->index_mask; 203 /* release so rxe can read it safely */ 204 smp_store_release(&q->buf->producer_index, prod); 205 break; 206 case QUEUE_TYPE_TO_ULP: 207 /* used by ulp, rxe owns the index */ 208 if (WARN_ON(1)) 209 pr_warn("%s: attempt to advance driver index\n", 210 __func__); 211 break; 212 } 213 } 214 215 static inline void queue_advance_consumer(struct rxe_queue *q, 216 enum queue_type type) 217 { 218 u32 cons; 219 220 switch (type) { 221 case QUEUE_TYPE_FROM_CLIENT: 222 /* used by rxe which owns the index */ 223 cons = (q->index + 1) & q->index_mask; 224 q->index = cons; 225 /* release so client can read it safely */ 226 smp_store_release(&q->buf->consumer_index, cons); 227 break; 228 case QUEUE_TYPE_TO_CLIENT: 229 /* used by rxe, client owns the index */ 230 if (WARN_ON(1)) 231 pr_warn("%s: attempt to advance client index\n", 232 __func__); 233 break; 234 case QUEUE_TYPE_FROM_ULP: 235 /* used by ulp, rxe owns the index */ 236 if (WARN_ON(1)) 237 pr_warn("%s: attempt to advance driver index\n", 238 __func__); 239 break; 240 case QUEUE_TYPE_TO_ULP: 241 /* used by ulp which owns the index */ 242 cons = q->buf->consumer_index; 243 cons = (cons + 1) & q->index_mask; 244 /* release so rxe can read it safely */ 245 smp_store_release(&q->buf->consumer_index, cons); 246 break; 247 } 248 } 249 250 static inline void *queue_producer_addr(struct rxe_queue *q, 251 enum queue_type type) 252 { 253 u32 prod = queue_get_producer(q, type); 254 255 return q->buf->data + (prod << q->log2_elem_size); 256 } 257 258 static inline void *queue_consumer_addr(struct rxe_queue *q, 259 enum queue_type type) 260 { 261 u32 cons = queue_get_consumer(q, type); 262 263 return q->buf->data + (cons << q->log2_elem_size); 264 } 265 266 static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index) 267 { 268 return q->buf->data + ((index & q->index_mask) 269 << q->log2_elem_size); 270 } 271 272 static inline u32 queue_index_from_addr(const struct rxe_queue *q, 273 const void *addr) 274 { 275 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) 276 & q->index_mask; 277 } 278 279 static inline void *queue_head(struct rxe_queue *q, enum queue_type type) 280 { 281 return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type); 282 } 283 284 #endif /* RXE_QUEUE_H */ 285