xref: /freebsd/contrib/ofed/libmlx4/srq.c (revision a90b9d0159070121c221b966469c3e36d912bf82) 
1 /*
2  * Copyright (c) 2007 Cisco, Inc.  All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <config.h>
34 
35 #include <stdlib.h>
36 #include <pthread.h>
37 #include <string.h>
38 
39 #include "mlx4.h"
40 #include "doorbell.h"
41 #include "wqe.h"
42 #include "mlx4-abi.h"
43 
44 static void *get_wqe(struct mlx4_srq *srq, int n)
45 {
46 	return srq->buf.buf + (n << srq->wqe_shift);
47 }
48 
49 void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind)
50 {
51 	struct mlx4_wqe_srq_next_seg *next;
52 
53 	pthread_spin_lock(&srq->lock);
54 
55 	next = get_wqe(srq, srq->tail);
56 	next->next_wqe_index = htobe16(ind);
57 	srq->tail = ind;
58 
59 	pthread_spin_unlock(&srq->lock);
60 }
61 
62 int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
63 		       struct ibv_recv_wr *wr,
64 		       struct ibv_recv_wr **bad_wr)
65 {
66 	struct mlx4_srq *srq = to_msrq(ibsrq);
67 	struct mlx4_wqe_srq_next_seg *next;
68 	struct mlx4_wqe_data_seg *scat;
69 	int err = 0;
70 	int nreq;
71 	int i;
72 
73 	pthread_spin_lock(&srq->lock);
74 
75 	for (nreq = 0; wr; ++nreq, wr = wr->next) {
76 		if (wr->num_sge > srq->max_gs) {
77 			err = -1;
78 			*bad_wr = wr;
79 			break;
80 		}
81 
82 		if (srq->head == srq->tail) {
83 			/* SRQ is full*/
84 			err = -1;
85 			*bad_wr = wr;
86 			break;
87 		}
88 
89 		srq->wrid[srq->head] = wr->wr_id;
90 
91 		next      = get_wqe(srq, srq->head);
92 		srq->head = be16toh(next->next_wqe_index);
93 		scat      = (struct mlx4_wqe_data_seg *) (next + 1);
94 
95 		for (i = 0; i < wr->num_sge; ++i) {
96 			scat[i].byte_count = htobe32(wr->sg_list[i].length);
97 			scat[i].lkey       = htobe32(wr->sg_list[i].lkey);
98 			scat[i].addr       = htobe64(wr->sg_list[i].addr);
99 		}
100 
101 		if (i < srq->max_gs) {
102 			scat[i].byte_count = 0;
103 			scat[i].lkey       = htobe32(MLX4_INVALID_LKEY);
104 			scat[i].addr       = 0;
105 		}
106 	}
107 
108 	if (nreq) {
109 		srq->counter += nreq;
110 
111 		/*
112 		 * Make sure that descriptors are written before
113 		 * we write doorbell record.
114 		 */
115 		udma_to_device_barrier();
116 
117 		*srq->db = htobe32(srq->counter);
118 	}
119 
120 	pthread_spin_unlock(&srq->lock);
121 
122 	return err;
123 }
124 
125 int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
126 		       struct mlx4_srq *srq)
127 {
128 	struct mlx4_wqe_srq_next_seg *next;
129 	struct mlx4_wqe_data_seg *scatter;
130 	int size;
131 	int buf_size;
132 	int i;
133 
134 	srq->wrid = malloc(srq->max * sizeof (uint64_t));
135 	if (!srq->wrid)
136 		return -1;
137 
138 	size = sizeof (struct mlx4_wqe_srq_next_seg) +
139 		srq->max_gs * sizeof (struct mlx4_wqe_data_seg);
140 
141 	for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift)
142 		; /* nothing */
143 
144 	buf_size = srq->max << srq->wqe_shift;
145 
146 	if (mlx4_alloc_buf(&srq->buf, buf_size,
147 			   to_mdev(pd->context->device)->page_size)) {
148 		free(srq->wrid);
149 		return -1;
150 	}
151 
152 	memset(srq->buf.buf, 0, buf_size);
153 
154 	/*
155 	 * Now initialize the SRQ buffer so that all of the WQEs are
156 	 * linked into the list of free WQEs.
157 	 */
158 
159 	for (i = 0; i < srq->max; ++i) {
160 		next = get_wqe(srq, i);
161 		next->next_wqe_index = htobe16((i + 1) & (srq->max - 1));
162 
163 		for (scatter = (void *) (next + 1);
164 		     (void *) scatter < (void *) next + (1 << srq->wqe_shift);
165 		     ++scatter)
166 			scatter->lkey = htobe32(MLX4_INVALID_LKEY);
167 	}
168 
169 	srq->head = 0;
170 	srq->tail = srq->max - 1;
171 
172 	return 0;
173 }
174 
175 void mlx4_cleanup_xsrq_table(struct mlx4_xsrq_table *xsrq_table)
176 {
177 	pthread_mutex_destroy(&xsrq_table->mutex);
178 }
179 
180 int mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size)
181 {
182 	int				ret;
183 	memset(xsrq_table, 0, sizeof *xsrq_table);
184 	xsrq_table->num_xsrq = size;
185 	xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS;
186 	xsrq_table->mask = (1 << xsrq_table->shift) - 1;
187 
188 	return pthread_mutex_init(&xsrq_table->mutex, NULL);
189 }
190 
191 struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn)
192 {
193 	int index;
194 
195 	index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
196 	if (xsrq_table->xsrq_table[index].refcnt)
197 		return xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask];
198 
199 	return NULL;
200 }
201 
202 int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn,
203 		    struct mlx4_srq *srq)
204 {
205 	int index, ret = 0;
206 
207 	index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
208 	pthread_mutex_lock(&xsrq_table->mutex);
209 	if (!xsrq_table->xsrq_table[index].refcnt) {
210 		xsrq_table->xsrq_table[index].table = calloc(xsrq_table->mask + 1,
211 							     sizeof(struct mlx4_srq *));
212 		if (!xsrq_table->xsrq_table[index].table) {
213 			ret = -1;
214 			goto out;
215 		}
216 	}
217 
218 	xsrq_table->xsrq_table[index].refcnt++;
219 	xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = srq;
220 
221 out:
222 	pthread_mutex_unlock(&xsrq_table->mutex);
223 	return ret;
224 }
225 
226 void mlx4_clear_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn)
227 {
228 	int index;
229 
230 	index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
231 	pthread_mutex_lock(&xsrq_table->mutex);
232 
233 	if (--xsrq_table->xsrq_table[index].refcnt)
234 		xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = NULL;
235 	else
236 		free(xsrq_table->xsrq_table[index].table);
237 
238 	pthread_mutex_unlock(&xsrq_table->mutex);
239 }
240 
241 struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context,
242 				    struct ibv_srq_init_attr_ex *attr_ex)
243 {
244 	struct mlx4_create_xsrq cmd;
245 	struct mlx4_create_srq_resp resp;
246 	struct mlx4_srq *srq;
247 	int ret;
248 
249 	/* Sanity check SRQ size before proceeding */
250 	if (attr_ex->attr.max_wr > 1 << 16 || attr_ex->attr.max_sge > 64)
251 		return NULL;
252 
253 	srq = calloc(1, sizeof *srq);
254 	if (!srq)
255 		return NULL;
256 
257 	if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
258 		goto err;
259 
260 	srq->max     = align_queue_size(attr_ex->attr.max_wr + 1);
261 	srq->max_gs  = attr_ex->attr.max_sge;
262 	srq->counter = 0;
263 	srq->ext_srq = 1;
264 
265 	if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq))
266 		goto err_spl;
267 
268 	srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ);
269 	if (!srq->db)
270 		goto err_free;
271 
272 	*srq->db = 0;
273 
274 	cmd.buf_addr = (uintptr_t) srq->buf.buf;
275 	cmd.db_addr  = (uintptr_t) srq->db;
276 
277 	ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq,
278 				    sizeof(srq->verbs_srq),
279 				    attr_ex,
280 				    &cmd.ibv_cmd, sizeof cmd,
281 				    &resp.ibv_resp, sizeof resp);
282 	if (ret)
283 		goto err_db;
284 
285 	ret = mlx4_store_xsrq(&to_mctx(context)->xsrq_table,
286 			      srq->verbs_srq.srq_num, srq);
287 	if (ret)
288 		goto err_destroy;
289 
290 	return &srq->verbs_srq.srq;
291 
292 err_destroy:
293 	ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
294 err_db:
295 	mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, srq->db);
296 err_free:
297 	free(srq->wrid);
298 	mlx4_free_buf(&srq->buf);
299 err_spl:
300 	pthread_spin_destroy(&srq->lock);
301 err:
302 	free(srq);
303 	return NULL;
304 }
305 
306 int mlx4_destroy_xrc_srq(struct ibv_srq *srq)
307 {
308 	struct mlx4_context *mctx = to_mctx(srq->context);
309 	struct mlx4_srq *msrq = to_msrq(srq);
310 	struct mlx4_cq *mcq;
311 	int ret;
312 
313 	mcq = to_mcq(msrq->verbs_srq.cq);
314 	mlx4_cq_clean(mcq, 0, msrq);
315 	pthread_spin_lock(&mcq->lock);
316 	mlx4_clear_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num);
317 	pthread_spin_unlock(&mcq->lock);
318 
319 	ret = ibv_cmd_destroy_srq(srq);
320 	if (ret) {
321 		pthread_spin_lock(&mcq->lock);
322 		mlx4_store_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num, msrq);
323 		pthread_spin_unlock(&mcq->lock);
324 		return ret;
325 	}
326 
327 	mlx4_free_db(mctx, MLX4_DB_TYPE_RQ, msrq->db);
328 	mlx4_free_buf(&msrq->buf);
329 	free(msrq->wrid);
330 	pthread_spin_destroy(&msrq->lock);
331 	free(msrq);
332 
333 	return 0;
334 }
335