1 /*
2 * Copyright (c) 2007 Cisco, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <config.h>
34
35 #include <stdlib.h>
36 #include <pthread.h>
37 #include <string.h>
38
39 #include "mlx4.h"
40 #include "doorbell.h"
41 #include "wqe.h"
42 #include "mlx4-abi.h"
43
get_wqe(struct mlx4_srq * srq,int n)44 static void *get_wqe(struct mlx4_srq *srq, int n)
45 {
46 return srq->buf.buf + (n << srq->wqe_shift);
47 }
48
mlx4_free_srq_wqe(struct mlx4_srq * srq,int ind)49 void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind)
50 {
51 struct mlx4_wqe_srq_next_seg *next;
52
53 pthread_spin_lock(&srq->lock);
54
55 next = get_wqe(srq, srq->tail);
56 next->next_wqe_index = htobe16(ind);
57 srq->tail = ind;
58
59 pthread_spin_unlock(&srq->lock);
60 }
61
mlx4_post_srq_recv(struct ibv_srq * ibsrq,struct ibv_recv_wr * wr,struct ibv_recv_wr ** bad_wr)62 int mlx4_post_srq_recv(struct ibv_srq *ibsrq,
63 struct ibv_recv_wr *wr,
64 struct ibv_recv_wr **bad_wr)
65 {
66 struct mlx4_srq *srq = to_msrq(ibsrq);
67 struct mlx4_wqe_srq_next_seg *next;
68 struct mlx4_wqe_data_seg *scat;
69 int err = 0;
70 int nreq;
71 int i;
72
73 pthread_spin_lock(&srq->lock);
74
75 for (nreq = 0; wr; ++nreq, wr = wr->next) {
76 if (wr->num_sge > srq->max_gs) {
77 err = -1;
78 *bad_wr = wr;
79 break;
80 }
81
82 if (srq->head == srq->tail) {
83 /* SRQ is full*/
84 err = -1;
85 *bad_wr = wr;
86 break;
87 }
88
89 srq->wrid[srq->head] = wr->wr_id;
90
91 next = get_wqe(srq, srq->head);
92 srq->head = be16toh(next->next_wqe_index);
93 scat = (struct mlx4_wqe_data_seg *) (next + 1);
94
95 for (i = 0; i < wr->num_sge; ++i) {
96 scat[i].byte_count = htobe32(wr->sg_list[i].length);
97 scat[i].lkey = htobe32(wr->sg_list[i].lkey);
98 scat[i].addr = htobe64(wr->sg_list[i].addr);
99 }
100
101 if (i < srq->max_gs) {
102 scat[i].byte_count = 0;
103 scat[i].lkey = htobe32(MLX4_INVALID_LKEY);
104 scat[i].addr = 0;
105 }
106 }
107
108 if (nreq) {
109 srq->counter += nreq;
110
111 /*
112 * Make sure that descriptors are written before
113 * we write doorbell record.
114 */
115 udma_to_device_barrier();
116
117 *srq->db = htobe32(srq->counter);
118 }
119
120 pthread_spin_unlock(&srq->lock);
121
122 return err;
123 }
124
mlx4_alloc_srq_buf(struct ibv_pd * pd,struct ibv_srq_attr * attr,struct mlx4_srq * srq)125 int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr,
126 struct mlx4_srq *srq)
127 {
128 struct mlx4_wqe_srq_next_seg *next;
129 struct mlx4_wqe_data_seg *scatter;
130 int size;
131 int buf_size;
132 int i;
133
134 srq->wrid = malloc(srq->max * sizeof (uint64_t));
135 if (!srq->wrid)
136 return -1;
137
138 size = sizeof (struct mlx4_wqe_srq_next_seg) +
139 srq->max_gs * sizeof (struct mlx4_wqe_data_seg);
140
141 for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift)
142 ; /* nothing */
143
144 buf_size = srq->max << srq->wqe_shift;
145
146 if (mlx4_alloc_buf(&srq->buf, buf_size,
147 to_mdev(pd->context->device)->page_size)) {
148 free(srq->wrid);
149 return -1;
150 }
151
152 memset(srq->buf.buf, 0, buf_size);
153
154 /*
155 * Now initialize the SRQ buffer so that all of the WQEs are
156 * linked into the list of free WQEs.
157 */
158
159 for (i = 0; i < srq->max; ++i) {
160 next = get_wqe(srq, i);
161 next->next_wqe_index = htobe16((i + 1) & (srq->max - 1));
162
163 for (scatter = (void *) (next + 1);
164 (void *) scatter < (void *) next + (1 << srq->wqe_shift);
165 ++scatter)
166 scatter->lkey = htobe32(MLX4_INVALID_LKEY);
167 }
168
169 srq->head = 0;
170 srq->tail = srq->max - 1;
171
172 return 0;
173 }
174
mlx4_cleanup_xsrq_table(struct mlx4_xsrq_table * xsrq_table)175 void mlx4_cleanup_xsrq_table(struct mlx4_xsrq_table *xsrq_table)
176 {
177 pthread_mutex_destroy(&xsrq_table->mutex);
178 }
179
mlx4_init_xsrq_table(struct mlx4_xsrq_table * xsrq_table,int size)180 int mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size)
181 {
182 int ret;
183 memset(xsrq_table, 0, sizeof *xsrq_table);
184 xsrq_table->num_xsrq = size;
185 xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS;
186 xsrq_table->mask = (1 << xsrq_table->shift) - 1;
187
188 return pthread_mutex_init(&xsrq_table->mutex, NULL);
189 }
190
mlx4_find_xsrq(struct mlx4_xsrq_table * xsrq_table,uint32_t srqn)191 struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn)
192 {
193 int index;
194
195 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
196 if (xsrq_table->xsrq_table[index].refcnt)
197 return xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask];
198
199 return NULL;
200 }
201
mlx4_store_xsrq(struct mlx4_xsrq_table * xsrq_table,uint32_t srqn,struct mlx4_srq * srq)202 int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn,
203 struct mlx4_srq *srq)
204 {
205 int index, ret = 0;
206
207 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
208 pthread_mutex_lock(&xsrq_table->mutex);
209 if (!xsrq_table->xsrq_table[index].refcnt) {
210 xsrq_table->xsrq_table[index].table = calloc(xsrq_table->mask + 1,
211 sizeof(struct mlx4_srq *));
212 if (!xsrq_table->xsrq_table[index].table) {
213 ret = -1;
214 goto out;
215 }
216 }
217
218 xsrq_table->xsrq_table[index].refcnt++;
219 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = srq;
220
221 out:
222 pthread_mutex_unlock(&xsrq_table->mutex);
223 return ret;
224 }
225
mlx4_clear_xsrq(struct mlx4_xsrq_table * xsrq_table,uint32_t srqn)226 void mlx4_clear_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn)
227 {
228 int index;
229
230 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift;
231 pthread_mutex_lock(&xsrq_table->mutex);
232
233 if (--xsrq_table->xsrq_table[index].refcnt)
234 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = NULL;
235 else
236 free(xsrq_table->xsrq_table[index].table);
237
238 pthread_mutex_unlock(&xsrq_table->mutex);
239 }
240
mlx4_create_xrc_srq(struct ibv_context * context,struct ibv_srq_init_attr_ex * attr_ex)241 struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context,
242 struct ibv_srq_init_attr_ex *attr_ex)
243 {
244 struct mlx4_create_xsrq cmd;
245 struct mlx4_create_srq_resp resp;
246 struct mlx4_srq *srq;
247 int ret;
248
249 /* Sanity check SRQ size before proceeding */
250 if (attr_ex->attr.max_wr > 1 << 16 || attr_ex->attr.max_sge > 64)
251 return NULL;
252
253 srq = calloc(1, sizeof *srq);
254 if (!srq)
255 return NULL;
256
257 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE))
258 goto err;
259
260 srq->max = align_queue_size(attr_ex->attr.max_wr + 1);
261 srq->max_gs = attr_ex->attr.max_sge;
262 srq->counter = 0;
263 srq->ext_srq = 1;
264
265 if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq))
266 goto err_spl;
267
268 srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ);
269 if (!srq->db)
270 goto err_free;
271
272 *srq->db = 0;
273
274 cmd.buf_addr = (uintptr_t) srq->buf.buf;
275 cmd.db_addr = (uintptr_t) srq->db;
276
277 ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq,
278 sizeof(srq->verbs_srq),
279 attr_ex,
280 &cmd.ibv_cmd, sizeof cmd,
281 &resp.ibv_resp, sizeof resp);
282 if (ret)
283 goto err_db;
284
285 ret = mlx4_store_xsrq(&to_mctx(context)->xsrq_table,
286 srq->verbs_srq.srq_num, srq);
287 if (ret)
288 goto err_destroy;
289
290 return &srq->verbs_srq.srq;
291
292 err_destroy:
293 ibv_cmd_destroy_srq(&srq->verbs_srq.srq);
294 err_db:
295 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, srq->db);
296 err_free:
297 free(srq->wrid);
298 mlx4_free_buf(&srq->buf);
299 err_spl:
300 pthread_spin_destroy(&srq->lock);
301 err:
302 free(srq);
303 return NULL;
304 }
305
mlx4_destroy_xrc_srq(struct ibv_srq * srq)306 int mlx4_destroy_xrc_srq(struct ibv_srq *srq)
307 {
308 struct mlx4_context *mctx = to_mctx(srq->context);
309 struct mlx4_srq *msrq = to_msrq(srq);
310 struct mlx4_cq *mcq;
311 int ret;
312
313 mcq = to_mcq(msrq->verbs_srq.cq);
314 mlx4_cq_clean(mcq, 0, msrq);
315 pthread_spin_lock(&mcq->lock);
316 mlx4_clear_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num);
317 pthread_spin_unlock(&mcq->lock);
318
319 ret = ibv_cmd_destroy_srq(srq);
320 if (ret) {
321 pthread_spin_lock(&mcq->lock);
322 mlx4_store_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num, msrq);
323 pthread_spin_unlock(&mcq->lock);
324 return ret;
325 }
326
327 mlx4_free_db(mctx, MLX4_DB_TYPE_RQ, msrq->db);
328 mlx4_free_buf(&msrq->buf);
329 free(msrq->wrid);
330 pthread_spin_destroy(&msrq->lock);
331 free(msrq);
332
333 return 0;
334 }
335