xref: /linux/drivers/infiniband/sw/rxe/rxe_queue.h (revision 9a87ffc99ec8eb8d35eed7c4f816d75f5cc9662e)
1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3  * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4  * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5  */
6 
7 #ifndef RXE_QUEUE_H
8 #define RXE_QUEUE_H
9 
10 /* Implements a simple circular buffer that is shared between user
11  * and the driver and can be resized. The requested element size is
12  * rounded up to a power of 2 and the number of elements in the buffer
13  * is also rounded up to a power of 2. Since the queue is empty when
14  * the producer and consumer indices match the maximum capacity of the
15  * queue is one less than the number of element slots.
16  *
17  * Notes:
18  *   - The driver indices are always masked off to q->index_mask
19  *     before storing so do not need to be checked on reads.
20  *   - The user whether user space or kernel is generally
21  *     not trusted so its parameters are masked to make sure
22  *     they do not access the queue out of bounds on reads.
23  *   - The driver indices for queues must not be written
24  *     by user so a local copy is used and a shared copy is
25  *     stored when the local copy is changed.
26  *   - By passing the type in the parameter list separate from q
27  *     the compiler can eliminate the switch statement when the
28  *     actual queue type is known when the function is called at
29  *     compile time.
30  *   - These queues are lock free. The user and driver must protect
31  *     changes to their end of the queues with locks if more than one
32  *     CPU can be accessing it at the same time.
33  */
34 
35 /**
36  * enum queue_type - type of queue
37  * @QUEUE_TYPE_TO_CLIENT:	Queue is written by rxe driver and
38  *				read by client which may be a user space
39  *				application or a kernel ulp.
40  *				Used by rxe internals only.
41  * @QUEUE_TYPE_FROM_CLIENT:	Queue is written by client and
42  *				read by rxe driver.
43  *				Used by rxe internals only.
44  * @QUEUE_TYPE_FROM_ULP:	Queue is written by kernel ulp and
45  *				read by rxe driver.
46  *				Used by kernel verbs APIs only on
47  *				behalf of ulps.
48  * @QUEUE_TYPE_TO_ULP:		Queue is written by rxe driver and
49  *				read by kernel ulp.
50  *				Used by kernel verbs APIs only on
51  *				behalf of ulps.
52  */
53 enum queue_type {
54 	QUEUE_TYPE_TO_CLIENT,
55 	QUEUE_TYPE_FROM_CLIENT,
56 	QUEUE_TYPE_FROM_ULP,
57 	QUEUE_TYPE_TO_ULP,
58 };
59 
60 struct rxe_queue_buf;
61 
62 struct rxe_queue {
63 	struct rxe_dev		*rxe;
64 	struct rxe_queue_buf	*buf;
65 	struct rxe_mmap_info	*ip;
66 	size_t			buf_size;
67 	size_t			elem_size;
68 	unsigned int		log2_elem_size;
69 	u32			index_mask;
70 	enum queue_type		type;
71 	/* private copy of index for shared queues between
72 	 * driver and clients. Driver reads and writes
73 	 * this copy and then replicates to rxe_queue_buf
74 	 * for read access by clients.
75 	 */
76 	u32			index;
77 };
78 
79 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
80 		 struct ib_udata *udata, struct rxe_queue_buf *buf,
81 		 size_t buf_size, struct rxe_mmap_info **ip_p);
82 
83 void rxe_queue_reset(struct rxe_queue *q);
84 
85 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
86 			unsigned int elem_size, enum queue_type type);
87 
88 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
89 		     unsigned int elem_size, struct ib_udata *udata,
90 		     struct mminfo __user *outbuf,
91 		     spinlock_t *producer_lock, spinlock_t *consumer_lock);
92 
93 void rxe_queue_cleanup(struct rxe_queue *queue);
94 
queue_next_index(struct rxe_queue * q,int index)95 static inline u32 queue_next_index(struct rxe_queue *q, int index)
96 {
97 	return (index + 1) & q->index_mask;
98 }
99 
queue_get_producer(const struct rxe_queue * q,enum queue_type type)100 static inline u32 queue_get_producer(const struct rxe_queue *q,
101 				     enum queue_type type)
102 {
103 	u32 prod;
104 
105 	switch (type) {
106 	case QUEUE_TYPE_FROM_CLIENT:
107 		/* used by rxe, client owns the index */
108 		prod = smp_load_acquire(&q->buf->producer_index);
109 		break;
110 	case QUEUE_TYPE_TO_CLIENT:
111 		/* used by rxe which owns the index */
112 		prod = q->index;
113 		break;
114 	case QUEUE_TYPE_FROM_ULP:
115 		/* used by ulp which owns the index */
116 		prod = q->buf->producer_index;
117 		break;
118 	case QUEUE_TYPE_TO_ULP:
119 		/* used by ulp, rxe owns the index */
120 		prod = smp_load_acquire(&q->buf->producer_index);
121 		break;
122 	}
123 
124 	return prod;
125 }
126 
queue_get_consumer(const struct rxe_queue * q,enum queue_type type)127 static inline u32 queue_get_consumer(const struct rxe_queue *q,
128 				     enum queue_type type)
129 {
130 	u32 cons;
131 
132 	switch (type) {
133 	case QUEUE_TYPE_FROM_CLIENT:
134 		/* used by rxe which owns the index */
135 		cons = q->index;
136 		break;
137 	case QUEUE_TYPE_TO_CLIENT:
138 		/* used by rxe, client owns the index */
139 		cons = smp_load_acquire(&q->buf->consumer_index);
140 		break;
141 	case QUEUE_TYPE_FROM_ULP:
142 		/* used by ulp, rxe owns the index */
143 		cons = smp_load_acquire(&q->buf->consumer_index);
144 		break;
145 	case QUEUE_TYPE_TO_ULP:
146 		/* used by ulp which owns the index */
147 		cons = q->buf->consumer_index;
148 		break;
149 	}
150 
151 	return cons;
152 }
153 
queue_empty(struct rxe_queue * q,enum queue_type type)154 static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
155 {
156 	u32 prod = queue_get_producer(q, type);
157 	u32 cons = queue_get_consumer(q, type);
158 
159 	return ((prod - cons) & q->index_mask) == 0;
160 }
161 
queue_full(struct rxe_queue * q,enum queue_type type)162 static inline int queue_full(struct rxe_queue *q, enum queue_type type)
163 {
164 	u32 prod = queue_get_producer(q, type);
165 	u32 cons = queue_get_consumer(q, type);
166 
167 	return ((prod + 1 - cons) & q->index_mask) == 0;
168 }
169 
queue_count(const struct rxe_queue * q,enum queue_type type)170 static inline u32 queue_count(const struct rxe_queue *q,
171 					enum queue_type type)
172 {
173 	u32 prod = queue_get_producer(q, type);
174 	u32 cons = queue_get_consumer(q, type);
175 
176 	return (prod - cons) & q->index_mask;
177 }
178 
queue_advance_producer(struct rxe_queue * q,enum queue_type type)179 static inline void queue_advance_producer(struct rxe_queue *q,
180 					  enum queue_type type)
181 {
182 	u32 prod;
183 
184 	switch (type) {
185 	case QUEUE_TYPE_FROM_CLIENT:
186 		/* used by rxe, client owns the index */
187 		if (WARN_ON(1))
188 			pr_warn("%s: attempt to advance client index\n",
189 				__func__);
190 		break;
191 	case QUEUE_TYPE_TO_CLIENT:
192 		/* used by rxe which owns the index */
193 		prod = q->index;
194 		prod = (prod + 1) & q->index_mask;
195 		q->index = prod;
196 		/* release so client can read it safely */
197 		smp_store_release(&q->buf->producer_index, prod);
198 		break;
199 	case QUEUE_TYPE_FROM_ULP:
200 		/* used by ulp which owns the index */
201 		prod = q->buf->producer_index;
202 		prod = (prod + 1) & q->index_mask;
203 		/* release so rxe can read it safely */
204 		smp_store_release(&q->buf->producer_index, prod);
205 		break;
206 	case QUEUE_TYPE_TO_ULP:
207 		/* used by ulp, rxe owns the index */
208 		if (WARN_ON(1))
209 			pr_warn("%s: attempt to advance driver index\n",
210 				__func__);
211 		break;
212 	}
213 }
214 
queue_advance_consumer(struct rxe_queue * q,enum queue_type type)215 static inline void queue_advance_consumer(struct rxe_queue *q,
216 					  enum queue_type type)
217 {
218 	u32 cons;
219 
220 	switch (type) {
221 	case QUEUE_TYPE_FROM_CLIENT:
222 		/* used by rxe which owns the index */
223 		cons = (q->index + 1) & q->index_mask;
224 		q->index = cons;
225 		/* release so client can read it safely */
226 		smp_store_release(&q->buf->consumer_index, cons);
227 		break;
228 	case QUEUE_TYPE_TO_CLIENT:
229 		/* used by rxe, client owns the index */
230 		if (WARN_ON(1))
231 			pr_warn("%s: attempt to advance client index\n",
232 				__func__);
233 		break;
234 	case QUEUE_TYPE_FROM_ULP:
235 		/* used by ulp, rxe owns the index */
236 		if (WARN_ON(1))
237 			pr_warn("%s: attempt to advance driver index\n",
238 				__func__);
239 		break;
240 	case QUEUE_TYPE_TO_ULP:
241 		/* used by ulp which owns the index */
242 		cons = q->buf->consumer_index;
243 		cons = (cons + 1) & q->index_mask;
244 		/* release so rxe can read it safely */
245 		smp_store_release(&q->buf->consumer_index, cons);
246 		break;
247 	}
248 }
249 
queue_producer_addr(struct rxe_queue * q,enum queue_type type)250 static inline void *queue_producer_addr(struct rxe_queue *q,
251 					enum queue_type type)
252 {
253 	u32 prod = queue_get_producer(q, type);
254 
255 	return q->buf->data + (prod << q->log2_elem_size);
256 }
257 
queue_consumer_addr(struct rxe_queue * q,enum queue_type type)258 static inline void *queue_consumer_addr(struct rxe_queue *q,
259 					enum queue_type type)
260 {
261 	u32 cons = queue_get_consumer(q, type);
262 
263 	return q->buf->data + (cons << q->log2_elem_size);
264 }
265 
queue_addr_from_index(struct rxe_queue * q,u32 index)266 static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
267 {
268 	return q->buf->data + ((index & q->index_mask)
269 				<< q->log2_elem_size);
270 }
271 
queue_index_from_addr(const struct rxe_queue * q,const void * addr)272 static inline u32 queue_index_from_addr(const struct rxe_queue *q,
273 				const void *addr)
274 {
275 	return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
276 				& q->index_mask;
277 }
278 
queue_head(struct rxe_queue * q,enum queue_type type)279 static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
280 {
281 	return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
282 }
283 
284 #endif /* RXE_QUEUE_H */
285