1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ 2 /* 3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved. 4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved. 5 */ 6 7 #ifndef RXE_QUEUE_H 8 #define RXE_QUEUE_H 9 10 /* for definition of shared struct rxe_queue_buf */ 11 #include <uapi/rdma/rdma_user_rxe.h> 12 13 /* implements a simple circular buffer that can optionally be 14 * shared between user space and the kernel and can be resized 15 * the requested element size is rounded up to a power of 2 16 * and the number of elements in the buffer is also rounded 17 * up to a power of 2. Since the queue is empty when the 18 * producer and consumer indices match the maximum capacity 19 * of the queue is one less than the number of element slots 20 */ 21 22 struct rxe_queue { 23 struct rxe_dev *rxe; 24 struct rxe_queue_buf *buf; 25 struct rxe_mmap_info *ip; 26 size_t buf_size; 27 size_t elem_size; 28 unsigned int log2_elem_size; 29 u32 index_mask; 30 }; 31 32 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf, 33 struct ib_udata *udata, struct rxe_queue_buf *buf, 34 size_t buf_size, struct rxe_mmap_info **ip_p); 35 36 void rxe_queue_reset(struct rxe_queue *q); 37 38 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, 39 int *num_elem, 40 unsigned int elem_size); 41 42 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p, 43 unsigned int elem_size, struct ib_udata *udata, 44 struct mminfo __user *outbuf, 45 /* Protect producers while resizing queue */ 46 spinlock_t *producer_lock, 47 /* Protect consumers while resizing queue */ 48 spinlock_t *consumer_lock); 49 50 void rxe_queue_cleanup(struct rxe_queue *queue); 51 52 static inline int next_index(struct rxe_queue *q, int index) 53 { 54 return (index + 1) & q->buf->index_mask; 55 } 56 57 static inline int queue_empty(struct rxe_queue *q) 58 { 59 u32 prod; 60 u32 cons; 61 62 /* make sure all changes to queue complete before 63 * testing queue empty 64 */ 65 prod = smp_load_acquire(&q->buf->producer_index); 66 /* same */ 67 cons = smp_load_acquire(&q->buf->consumer_index); 68 69 return ((prod - cons) & q->index_mask) == 0; 70 } 71 72 static inline int queue_full(struct rxe_queue *q) 73 { 74 u32 prod; 75 u32 cons; 76 77 /* make sure all changes to queue complete before 78 * testing queue full 79 */ 80 prod = smp_load_acquire(&q->buf->producer_index); 81 /* same */ 82 cons = smp_load_acquire(&q->buf->consumer_index); 83 84 return ((prod + 1 - cons) & q->index_mask) == 0; 85 } 86 87 static inline void advance_producer(struct rxe_queue *q) 88 { 89 u32 prod; 90 91 prod = (q->buf->producer_index + 1) & q->index_mask; 92 93 /* make sure all changes to queue complete before 94 * changing producer index 95 */ 96 smp_store_release(&q->buf->producer_index, prod); 97 } 98 99 static inline void advance_consumer(struct rxe_queue *q) 100 { 101 u32 cons; 102 103 cons = (q->buf->consumer_index + 1) & q->index_mask; 104 105 /* make sure all changes to queue complete before 106 * changing consumer index 107 */ 108 smp_store_release(&q->buf->consumer_index, cons); 109 } 110 111 static inline void *producer_addr(struct rxe_queue *q) 112 { 113 return q->buf->data + ((q->buf->producer_index & q->index_mask) 114 << q->log2_elem_size); 115 } 116 117 static inline void *consumer_addr(struct rxe_queue *q) 118 { 119 return q->buf->data + ((q->buf->consumer_index & q->index_mask) 120 << q->log2_elem_size); 121 } 122 123 static inline unsigned int producer_index(struct rxe_queue *q) 124 { 125 u32 index; 126 127 /* make sure all changes to queue 128 * complete before getting producer index 129 */ 130 index = smp_load_acquire(&q->buf->producer_index); 131 index &= q->index_mask; 132 133 return index; 134 } 135 136 static inline unsigned int consumer_index(struct rxe_queue *q) 137 { 138 u32 index; 139 140 /* make sure all changes to queue 141 * complete before getting consumer index 142 */ 143 index = smp_load_acquire(&q->buf->consumer_index); 144 index &= q->index_mask; 145 146 return index; 147 } 148 149 static inline void *addr_from_index(struct rxe_queue *q, unsigned int index) 150 { 151 return q->buf->data + ((index & q->index_mask) 152 << q->buf->log2_elem_size); 153 } 154 155 static inline unsigned int index_from_addr(const struct rxe_queue *q, 156 const void *addr) 157 { 158 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size) 159 & q->index_mask; 160 } 161 162 static inline unsigned int queue_count(const struct rxe_queue *q) 163 { 164 return (q->buf->producer_index - q->buf->consumer_index) 165 & q->index_mask; 166 } 167 168 static inline void *queue_head(struct rxe_queue *q) 169 { 170 return queue_empty(q) ? NULL : consumer_addr(q); 171 } 172 173 #endif /* RXE_QUEUE_H */ 174