1 /* 2 * Copyright (c) 2007 Cisco, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <config.h> 34 35 #include <stdlib.h> 36 #include <pthread.h> 37 #include <string.h> 38 39 #include "mlx4.h" 40 #include "doorbell.h" 41 #include "wqe.h" 42 #include "mlx4-abi.h" 43 44 static void *get_wqe(struct mlx4_srq *srq, int n) 45 { 46 return srq->buf.buf + (n << srq->wqe_shift); 47 } 48 49 void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind) 50 { 51 struct mlx4_wqe_srq_next_seg *next; 52 53 pthread_spin_lock(&srq->lock); 54 55 next = get_wqe(srq, srq->tail); 56 next->next_wqe_index = htobe16(ind); 57 srq->tail = ind; 58 59 pthread_spin_unlock(&srq->lock); 60 } 61 62 int mlx4_post_srq_recv(struct ibv_srq *ibsrq, 63 struct ibv_recv_wr *wr, 64 struct ibv_recv_wr **bad_wr) 65 { 66 struct mlx4_srq *srq = to_msrq(ibsrq); 67 struct mlx4_wqe_srq_next_seg *next; 68 struct mlx4_wqe_data_seg *scat; 69 int err = 0; 70 int nreq; 71 int i; 72 73 pthread_spin_lock(&srq->lock); 74 75 for (nreq = 0; wr; ++nreq, wr = wr->next) { 76 if (wr->num_sge > srq->max_gs) { 77 err = -1; 78 *bad_wr = wr; 79 break; 80 } 81 82 if (srq->head == srq->tail) { 83 /* SRQ is full*/ 84 err = -1; 85 *bad_wr = wr; 86 break; 87 } 88 89 srq->wrid[srq->head] = wr->wr_id; 90 91 next = get_wqe(srq, srq->head); 92 srq->head = be16toh(next->next_wqe_index); 93 scat = (struct mlx4_wqe_data_seg *) (next + 1); 94 95 for (i = 0; i < wr->num_sge; ++i) { 96 scat[i].byte_count = htobe32(wr->sg_list[i].length); 97 scat[i].lkey = htobe32(wr->sg_list[i].lkey); 98 scat[i].addr = htobe64(wr->sg_list[i].addr); 99 } 100 101 if (i < srq->max_gs) { 102 scat[i].byte_count = 0; 103 scat[i].lkey = htobe32(MLX4_INVALID_LKEY); 104 scat[i].addr = 0; 105 } 106 } 107 108 if (nreq) { 109 srq->counter += nreq; 110 111 /* 112 * Make sure that descriptors are written before 113 * we write doorbell record. 114 */ 115 udma_to_device_barrier(); 116 117 *srq->db = htobe32(srq->counter); 118 } 119 120 pthread_spin_unlock(&srq->lock); 121 122 return err; 123 } 124 125 int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 126 struct mlx4_srq *srq) 127 { 128 struct mlx4_wqe_srq_next_seg *next; 129 struct mlx4_wqe_data_seg *scatter; 130 int size; 131 int buf_size; 132 int i; 133 134 srq->wrid = malloc(srq->max * sizeof (uint64_t)); 135 if (!srq->wrid) 136 return -1; 137 138 size = sizeof (struct mlx4_wqe_srq_next_seg) + 139 srq->max_gs * sizeof (struct mlx4_wqe_data_seg); 140 141 for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift) 142 ; /* nothing */ 143 144 buf_size = srq->max << srq->wqe_shift; 145 146 if (mlx4_alloc_buf(&srq->buf, buf_size, 147 to_mdev(pd->context->device)->page_size)) { 148 free(srq->wrid); 149 return -1; 150 } 151 152 memset(srq->buf.buf, 0, buf_size); 153 154 /* 155 * Now initialize the SRQ buffer so that all of the WQEs are 156 * linked into the list of free WQEs. 157 */ 158 159 for (i = 0; i < srq->max; ++i) { 160 next = get_wqe(srq, i); 161 next->next_wqe_index = htobe16((i + 1) & (srq->max - 1)); 162 163 for (scatter = (void *) (next + 1); 164 (void *) scatter < (void *) next + (1 << srq->wqe_shift); 165 ++scatter) 166 scatter->lkey = htobe32(MLX4_INVALID_LKEY); 167 } 168 169 srq->head = 0; 170 srq->tail = srq->max - 1; 171 172 return 0; 173 } 174 175 void mlx4_cleanup_xsrq_table(struct mlx4_xsrq_table *xsrq_table) 176 { 177 pthread_mutex_destroy(&xsrq_table->mutex); 178 } 179 180 int mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size) 181 { 182 int ret; 183 memset(xsrq_table, 0, sizeof *xsrq_table); 184 xsrq_table->num_xsrq = size; 185 xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS; 186 xsrq_table->mask = (1 << xsrq_table->shift) - 1; 187 188 return pthread_mutex_init(&xsrq_table->mutex, NULL); 189 } 190 191 struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn) 192 { 193 int index; 194 195 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; 196 if (xsrq_table->xsrq_table[index].refcnt) 197 return xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask]; 198 199 return NULL; 200 } 201 202 int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn, 203 struct mlx4_srq *srq) 204 { 205 int index, ret = 0; 206 207 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; 208 pthread_mutex_lock(&xsrq_table->mutex); 209 if (!xsrq_table->xsrq_table[index].refcnt) { 210 xsrq_table->xsrq_table[index].table = calloc(xsrq_table->mask + 1, 211 sizeof(struct mlx4_srq *)); 212 if (!xsrq_table->xsrq_table[index].table) { 213 ret = -1; 214 goto out; 215 } 216 } 217 218 xsrq_table->xsrq_table[index].refcnt++; 219 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = srq; 220 221 out: 222 pthread_mutex_unlock(&xsrq_table->mutex); 223 return ret; 224 } 225 226 void mlx4_clear_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn) 227 { 228 int index; 229 230 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; 231 pthread_mutex_lock(&xsrq_table->mutex); 232 233 if (--xsrq_table->xsrq_table[index].refcnt) 234 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = NULL; 235 else 236 free(xsrq_table->xsrq_table[index].table); 237 238 pthread_mutex_unlock(&xsrq_table->mutex); 239 } 240 241 struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context, 242 struct ibv_srq_init_attr_ex *attr_ex) 243 { 244 struct mlx4_create_xsrq cmd; 245 struct mlx4_create_srq_resp resp; 246 struct mlx4_srq *srq; 247 int ret; 248 249 /* Sanity check SRQ size before proceeding */ 250 if (attr_ex->attr.max_wr > 1 << 16 || attr_ex->attr.max_sge > 64) 251 return NULL; 252 253 srq = calloc(1, sizeof *srq); 254 if (!srq) 255 return NULL; 256 257 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) 258 goto err; 259 260 srq->max = align_queue_size(attr_ex->attr.max_wr + 1); 261 srq->max_gs = attr_ex->attr.max_sge; 262 srq->counter = 0; 263 srq->ext_srq = 1; 264 265 if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq)) 266 goto err_spl; 267 268 srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ); 269 if (!srq->db) 270 goto err_free; 271 272 *srq->db = 0; 273 274 cmd.buf_addr = (uintptr_t) srq->buf.buf; 275 cmd.db_addr = (uintptr_t) srq->db; 276 277 ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, 278 sizeof(srq->verbs_srq), 279 attr_ex, 280 &cmd.ibv_cmd, sizeof cmd, 281 &resp.ibv_resp, sizeof resp); 282 if (ret) 283 goto err_db; 284 285 ret = mlx4_store_xsrq(&to_mctx(context)->xsrq_table, 286 srq->verbs_srq.srq_num, srq); 287 if (ret) 288 goto err_destroy; 289 290 return &srq->verbs_srq.srq; 291 292 err_destroy: 293 ibv_cmd_destroy_srq(&srq->verbs_srq.srq); 294 err_db: 295 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, srq->db); 296 err_free: 297 free(srq->wrid); 298 mlx4_free_buf(&srq->buf); 299 err_spl: 300 pthread_spin_destroy(&srq->lock); 301 err: 302 free(srq); 303 return NULL; 304 } 305 306 int mlx4_destroy_xrc_srq(struct ibv_srq *srq) 307 { 308 struct mlx4_context *mctx = to_mctx(srq->context); 309 struct mlx4_srq *msrq = to_msrq(srq); 310 struct mlx4_cq *mcq; 311 int ret; 312 313 mcq = to_mcq(msrq->verbs_srq.cq); 314 mlx4_cq_clean(mcq, 0, msrq); 315 pthread_spin_lock(&mcq->lock); 316 mlx4_clear_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num); 317 pthread_spin_unlock(&mcq->lock); 318 319 ret = ibv_cmd_destroy_srq(srq); 320 if (ret) { 321 pthread_spin_lock(&mcq->lock); 322 mlx4_store_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num, msrq); 323 pthread_spin_unlock(&mcq->lock); 324 return ret; 325 } 326 327 mlx4_free_db(mctx, MLX4_DB_TYPE_RQ, msrq->db); 328 mlx4_free_buf(&msrq->buf); 329 free(msrq->wrid); 330 pthread_spin_destroy(&msrq->lock); 331 free(msrq); 332 333 return 0; 334 } 335