163fa15dbSBob Pearson /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
28700e3e7SMoni Shoua /*
38700e3e7SMoni Shoua * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
48700e3e7SMoni Shoua * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
58700e3e7SMoni Shoua */
68700e3e7SMoni Shoua
78700e3e7SMoni Shoua #ifndef RXE_QUEUE_H
88700e3e7SMoni Shoua #define RXE_QUEUE_H
98700e3e7SMoni Shoua
10ae6e843fSBob Pearson /* Implements a simple circular buffer that is shared between user
11ae6e843fSBob Pearson * and the driver and can be resized. The requested element size is
12ae6e843fSBob Pearson * rounded up to a power of 2 and the number of elements in the buffer
13ae6e843fSBob Pearson * is also rounded up to a power of 2. Since the queue is empty when
14ae6e843fSBob Pearson * the producer and consumer indices match the maximum capacity of the
15ae6e843fSBob Pearson * queue is one less than the number of element slots.
165bcf5a59SBob Pearson *
175bcf5a59SBob Pearson * Notes:
18ae6e843fSBob Pearson * - The driver indices are always masked off to q->index_mask
195bcf5a59SBob Pearson * before storing so do not need to be checked on reads.
20ae6e843fSBob Pearson * - The user whether user space or kernel is generally
21ae6e843fSBob Pearson * not trusted so its parameters are masked to make sure
22ae6e843fSBob Pearson * they do not access the queue out of bounds on reads.
23ae6e843fSBob Pearson * - The driver indices for queues must not be written
24ae6e843fSBob Pearson * by user so a local copy is used and a shared copy is
25ae6e843fSBob Pearson * stored when the local copy is changed.
265bcf5a59SBob Pearson * - By passing the type in the parameter list separate from q
275bcf5a59SBob Pearson * the compiler can eliminate the switch statement when the
28ae6e843fSBob Pearson * actual queue type is known when the function is called at
29ae6e843fSBob Pearson * compile time.
30ae6e843fSBob Pearson * - These queues are lock free. The user and driver must protect
31ae6e843fSBob Pearson * changes to their end of the queues with locks if more than one
32ae6e843fSBob Pearson * CPU can be accessing it at the same time.
338700e3e7SMoni Shoua */
348700e3e7SMoni Shoua
35ae6e843fSBob Pearson /**
36ae6e843fSBob Pearson * enum queue_type - type of queue
37ae6e843fSBob Pearson * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and
38*a77a5238SBob Pearson * read by client which may be a user space
39*a77a5238SBob Pearson * application or a kernel ulp.
40*a77a5238SBob Pearson * Used by rxe internals only.
41ae6e843fSBob Pearson * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and
42*a77a5238SBob Pearson * read by rxe driver.
43*a77a5238SBob Pearson * Used by rxe internals only.
44*a77a5238SBob Pearson * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and
45*a77a5238SBob Pearson * read by rxe driver.
46*a77a5238SBob Pearson * Used by kernel verbs APIs only on
47*a77a5238SBob Pearson * behalf of ulps.
48*a77a5238SBob Pearson * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and
49*a77a5238SBob Pearson * read by kernel ulp.
50*a77a5238SBob Pearson * Used by kernel verbs APIs only on
51*a77a5238SBob Pearson * behalf of ulps.
52ae6e843fSBob Pearson */
5359daff49SBob Pearson enum queue_type {
54ae6e843fSBob Pearson QUEUE_TYPE_TO_CLIENT,
55ae6e843fSBob Pearson QUEUE_TYPE_FROM_CLIENT,
56*a77a5238SBob Pearson QUEUE_TYPE_FROM_ULP,
57*a77a5238SBob Pearson QUEUE_TYPE_TO_ULP,
5859daff49SBob Pearson };
5959daff49SBob Pearson
60f5d1f6d6SBob Pearson struct rxe_queue_buf;
61f5d1f6d6SBob Pearson
628700e3e7SMoni Shoua struct rxe_queue {
638700e3e7SMoni Shoua struct rxe_dev *rxe;
648700e3e7SMoni Shoua struct rxe_queue_buf *buf;
658700e3e7SMoni Shoua struct rxe_mmap_info *ip;
668700e3e7SMoni Shoua size_t buf_size;
678700e3e7SMoni Shoua size_t elem_size;
688700e3e7SMoni Shoua unsigned int log2_elem_size;
69d21a1240SBob Pearson u32 index_mask;
7059daff49SBob Pearson enum queue_type type;
715bcf5a59SBob Pearson /* private copy of index for shared queues between
72*a77a5238SBob Pearson * driver and clients. Driver reads and writes
735bcf5a59SBob Pearson * this copy and then replicates to rxe_queue_buf
74*a77a5238SBob Pearson * for read access by clients.
755bcf5a59SBob Pearson */
765bcf5a59SBob Pearson u32 index;
778700e3e7SMoni Shoua };
788700e3e7SMoni Shoua
79ff23dfa1SShamir Rabinovitch int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
80ff23dfa1SShamir Rabinovitch struct ib_udata *udata, struct rxe_queue_buf *buf,
81ff23dfa1SShamir Rabinovitch size_t buf_size, struct rxe_mmap_info **ip_p);
828700e3e7SMoni Shoua
83aa75b07bSYonatan Cohen void rxe_queue_reset(struct rxe_queue *q);
84aa75b07bSYonatan Cohen
8559daff49SBob Pearson struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
8659daff49SBob Pearson unsigned int elem_size, enum queue_type type);
878700e3e7SMoni Shoua
88ff23dfa1SShamir Rabinovitch int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
89ff23dfa1SShamir Rabinovitch unsigned int elem_size, struct ib_udata *udata,
900c43ab37SJason Gunthorpe struct mminfo __user *outbuf,
91ae6e843fSBob Pearson spinlock_t *producer_lock, spinlock_t *consumer_lock);
928700e3e7SMoni Shoua
938700e3e7SMoni Shoua void rxe_queue_cleanup(struct rxe_queue *queue);
948700e3e7SMoni Shoua
queue_next_index(struct rxe_queue * q,int index)95ae6e843fSBob Pearson static inline u32 queue_next_index(struct rxe_queue *q, int index)
968700e3e7SMoni Shoua {
97ae6e843fSBob Pearson return (index + 1) & q->index_mask;
988700e3e7SMoni Shoua }
998700e3e7SMoni Shoua
queue_get_producer(const struct rxe_queue * q,enum queue_type type)100ae6e843fSBob Pearson static inline u32 queue_get_producer(const struct rxe_queue *q,
1015bcf5a59SBob Pearson enum queue_type type)
1028700e3e7SMoni Shoua {
1030a67c46dSBob Pearson u32 prod;
104d21a1240SBob Pearson
1055bcf5a59SBob Pearson switch (type) {
106ae6e843fSBob Pearson case QUEUE_TYPE_FROM_CLIENT:
107*a77a5238SBob Pearson /* used by rxe, client owns the index */
1080a67c46dSBob Pearson prod = smp_load_acquire(&q->buf->producer_index);
1095bcf5a59SBob Pearson break;
110ae6e843fSBob Pearson case QUEUE_TYPE_TO_CLIENT:
111*a77a5238SBob Pearson /* used by rxe which owns the index */
1125bcf5a59SBob Pearson prod = q->index;
1135bcf5a59SBob Pearson break;
114*a77a5238SBob Pearson case QUEUE_TYPE_FROM_ULP:
115*a77a5238SBob Pearson /* used by ulp which owns the index */
1165bcf5a59SBob Pearson prod = q->buf->producer_index;
1175bcf5a59SBob Pearson break;
118*a77a5238SBob Pearson case QUEUE_TYPE_TO_ULP:
119*a77a5238SBob Pearson /* used by ulp, rxe owns the index */
120*a77a5238SBob Pearson prod = smp_load_acquire(&q->buf->producer_index);
121*a77a5238SBob Pearson break;
1225bcf5a59SBob Pearson }
1230a67c46dSBob Pearson
1240a67c46dSBob Pearson return prod;
1258700e3e7SMoni Shoua }
1268700e3e7SMoni Shoua
queue_get_consumer(const struct rxe_queue * q,enum queue_type type)127ae6e843fSBob Pearson static inline u32 queue_get_consumer(const struct rxe_queue *q,
1285bcf5a59SBob Pearson enum queue_type type)
1298700e3e7SMoni Shoua {
1300a67c46dSBob Pearson u32 cons;
131d21a1240SBob Pearson
1325bcf5a59SBob Pearson switch (type) {
133ae6e843fSBob Pearson case QUEUE_TYPE_FROM_CLIENT:
134*a77a5238SBob Pearson /* used by rxe which owns the index */
1355bcf5a59SBob Pearson cons = q->index;
1365bcf5a59SBob Pearson break;
137ae6e843fSBob Pearson case QUEUE_TYPE_TO_CLIENT:
138*a77a5238SBob Pearson /* used by rxe, client owns the index */
1390a67c46dSBob Pearson cons = smp_load_acquire(&q->buf->consumer_index);
1405bcf5a59SBob Pearson break;
141*a77a5238SBob Pearson case QUEUE_TYPE_FROM_ULP:
142*a77a5238SBob Pearson /* used by ulp, rxe owns the index */
143*a77a5238SBob Pearson cons = smp_load_acquire(&q->buf->consumer_index);
144*a77a5238SBob Pearson break;
145*a77a5238SBob Pearson case QUEUE_TYPE_TO_ULP:
146*a77a5238SBob Pearson /* used by ulp which owns the index */
1475bcf5a59SBob Pearson cons = q->buf->consumer_index;
1485bcf5a59SBob Pearson break;
1495bcf5a59SBob Pearson }
1500a67c46dSBob Pearson
1510a67c46dSBob Pearson return cons;
1528700e3e7SMoni Shoua }
1538700e3e7SMoni Shoua
queue_empty(struct rxe_queue * q,enum queue_type type)154ae6e843fSBob Pearson static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
1558700e3e7SMoni Shoua {
156ae6e843fSBob Pearson u32 prod = queue_get_producer(q, type);
157ae6e843fSBob Pearson u32 cons = queue_get_consumer(q, type);
158ae6e843fSBob Pearson
159ae6e843fSBob Pearson return ((prod - cons) & q->index_mask) == 0;
1608700e3e7SMoni Shoua }
1618700e3e7SMoni Shoua
queue_full(struct rxe_queue * q,enum queue_type type)162ae6e843fSBob Pearson static inline int queue_full(struct rxe_queue *q, enum queue_type type)
163ae6e843fSBob Pearson {
164ae6e843fSBob Pearson u32 prod = queue_get_producer(q, type);
165ae6e843fSBob Pearson u32 cons = queue_get_consumer(q, type);
166ae6e843fSBob Pearson
167ae6e843fSBob Pearson return ((prod + 1 - cons) & q->index_mask) == 0;
168ae6e843fSBob Pearson }
169ae6e843fSBob Pearson
queue_count(const struct rxe_queue * q,enum queue_type type)170ae6e843fSBob Pearson static inline u32 queue_count(const struct rxe_queue *q,
171ae6e843fSBob Pearson enum queue_type type)
172ae6e843fSBob Pearson {
173ae6e843fSBob Pearson u32 prod = queue_get_producer(q, type);
174ae6e843fSBob Pearson u32 cons = queue_get_consumer(q, type);
175ae6e843fSBob Pearson
176ae6e843fSBob Pearson return (prod - cons) & q->index_mask;
177ae6e843fSBob Pearson }
178ae6e843fSBob Pearson
queue_advance_producer(struct rxe_queue * q,enum queue_type type)179ae6e843fSBob Pearson static inline void queue_advance_producer(struct rxe_queue *q,
180ae6e843fSBob Pearson enum queue_type type)
181ae6e843fSBob Pearson {
182ae6e843fSBob Pearson u32 prod;
183ae6e843fSBob Pearson
184ae6e843fSBob Pearson switch (type) {
185ae6e843fSBob Pearson case QUEUE_TYPE_FROM_CLIENT:
186*a77a5238SBob Pearson /* used by rxe, client owns the index */
187*a77a5238SBob Pearson if (WARN_ON(1))
188ae6e843fSBob Pearson pr_warn("%s: attempt to advance client index\n",
189ae6e843fSBob Pearson __func__);
190ae6e843fSBob Pearson break;
191ae6e843fSBob Pearson case QUEUE_TYPE_TO_CLIENT:
192*a77a5238SBob Pearson /* used by rxe which owns the index */
193ae6e843fSBob Pearson prod = q->index;
194ae6e843fSBob Pearson prod = (prod + 1) & q->index_mask;
195ae6e843fSBob Pearson q->index = prod;
196*a77a5238SBob Pearson /* release so client can read it safely */
197ae6e843fSBob Pearson smp_store_release(&q->buf->producer_index, prod);
198ae6e843fSBob Pearson break;
199*a77a5238SBob Pearson case QUEUE_TYPE_FROM_ULP:
200*a77a5238SBob Pearson /* used by ulp which owns the index */
201ae6e843fSBob Pearson prod = q->buf->producer_index;
202ae6e843fSBob Pearson prod = (prod + 1) & q->index_mask;
203*a77a5238SBob Pearson /* release so rxe can read it safely */
204*a77a5238SBob Pearson smp_store_release(&q->buf->producer_index, prod);
205*a77a5238SBob Pearson break;
206*a77a5238SBob Pearson case QUEUE_TYPE_TO_ULP:
207*a77a5238SBob Pearson /* used by ulp, rxe owns the index */
208*a77a5238SBob Pearson if (WARN_ON(1))
209*a77a5238SBob Pearson pr_warn("%s: attempt to advance driver index\n",
210*a77a5238SBob Pearson __func__);
211ae6e843fSBob Pearson break;
212ae6e843fSBob Pearson }
213ae6e843fSBob Pearson }
214ae6e843fSBob Pearson
queue_advance_consumer(struct rxe_queue * q,enum queue_type type)215ae6e843fSBob Pearson static inline void queue_advance_consumer(struct rxe_queue *q,
216ae6e843fSBob Pearson enum queue_type type)
217ae6e843fSBob Pearson {
218ae6e843fSBob Pearson u32 cons;
219ae6e843fSBob Pearson
220ae6e843fSBob Pearson switch (type) {
221ae6e843fSBob Pearson case QUEUE_TYPE_FROM_CLIENT:
222*a77a5238SBob Pearson /* used by rxe which owns the index */
223*a77a5238SBob Pearson cons = (q->index + 1) & q->index_mask;
224ae6e843fSBob Pearson q->index = cons;
225*a77a5238SBob Pearson /* release so client can read it safely */
226ae6e843fSBob Pearson smp_store_release(&q->buf->consumer_index, cons);
227ae6e843fSBob Pearson break;
228ae6e843fSBob Pearson case QUEUE_TYPE_TO_CLIENT:
229*a77a5238SBob Pearson /* used by rxe, client owns the index */
230*a77a5238SBob Pearson if (WARN_ON(1))
231ae6e843fSBob Pearson pr_warn("%s: attempt to advance client index\n",
232ae6e843fSBob Pearson __func__);
233ae6e843fSBob Pearson break;
234*a77a5238SBob Pearson case QUEUE_TYPE_FROM_ULP:
235*a77a5238SBob Pearson /* used by ulp, rxe owns the index */
236*a77a5238SBob Pearson if (WARN_ON(1))
237ae6e843fSBob Pearson pr_warn("%s: attempt to advance driver index\n",
238ae6e843fSBob Pearson __func__);
239ae6e843fSBob Pearson break;
240*a77a5238SBob Pearson case QUEUE_TYPE_TO_ULP:
241*a77a5238SBob Pearson /* used by ulp which owns the index */
242*a77a5238SBob Pearson cons = q->buf->consumer_index;
243*a77a5238SBob Pearson cons = (cons + 1) & q->index_mask;
244*a77a5238SBob Pearson /* release so rxe can read it safely */
245*a77a5238SBob Pearson smp_store_release(&q->buf->consumer_index, cons);
246*a77a5238SBob Pearson break;
247ae6e843fSBob Pearson }
248ae6e843fSBob Pearson }
249ae6e843fSBob Pearson
queue_producer_addr(struct rxe_queue * q,enum queue_type type)250ae6e843fSBob Pearson static inline void *queue_producer_addr(struct rxe_queue *q,
251ae6e843fSBob Pearson enum queue_type type)
252ae6e843fSBob Pearson {
253ae6e843fSBob Pearson u32 prod = queue_get_producer(q, type);
254ae6e843fSBob Pearson
255ae6e843fSBob Pearson return q->buf->data + (prod << q->log2_elem_size);
256ae6e843fSBob Pearson }
257ae6e843fSBob Pearson
queue_consumer_addr(struct rxe_queue * q,enum queue_type type)258ae6e843fSBob Pearson static inline void *queue_consumer_addr(struct rxe_queue *q,
259ae6e843fSBob Pearson enum queue_type type)
260ae6e843fSBob Pearson {
261ae6e843fSBob Pearson u32 cons = queue_get_consumer(q, type);
262ae6e843fSBob Pearson
263ae6e843fSBob Pearson return q->buf->data + (cons << q->log2_elem_size);
264ae6e843fSBob Pearson }
265ae6e843fSBob Pearson
queue_addr_from_index(struct rxe_queue * q,u32 index)266ae6e843fSBob Pearson static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
267ae6e843fSBob Pearson {
268ae6e843fSBob Pearson return q->buf->data + ((index & q->index_mask)
269ae6e843fSBob Pearson << q->log2_elem_size);
270ae6e843fSBob Pearson }
271ae6e843fSBob Pearson
queue_index_from_addr(const struct rxe_queue * q,const void * addr)272ae6e843fSBob Pearson static inline u32 queue_index_from_addr(const struct rxe_queue *q,
2738700e3e7SMoni Shoua const void *addr)
2748700e3e7SMoni Shoua {
2758700e3e7SMoni Shoua return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
2768700e3e7SMoni Shoua & q->index_mask;
2778700e3e7SMoni Shoua }
2788700e3e7SMoni Shoua
queue_head(struct rxe_queue * q,enum queue_type type)2795bcf5a59SBob Pearson static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
2808700e3e7SMoni Shoua {
281ae6e843fSBob Pearson return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
2828700e3e7SMoni Shoua }
2838700e3e7SMoni Shoua
2848700e3e7SMoni Shoua #endif /* RXE_QUEUE_H */
285