1 /* 2 * Copyright (c) 2007 Cisco, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <config.h> 34 35 #include <stdlib.h> 36 #include <pthread.h> 37 #include <string.h> 38 39 #include "mlx4.h" 40 #include "doorbell.h" 41 #include "wqe.h" 42 #include "mlx4-abi.h" 43 44 static void *get_wqe(struct mlx4_srq *srq, int n) 45 { 46 return srq->buf.buf + (n << srq->wqe_shift); 47 } 48 49 void mlx4_free_srq_wqe(struct mlx4_srq *srq, int ind) 50 { 51 struct mlx4_wqe_srq_next_seg *next; 52 53 pthread_spin_lock(&srq->lock); 54 55 next = get_wqe(srq, srq->tail); 56 next->next_wqe_index = htobe16(ind); 57 srq->tail = ind; 58 59 pthread_spin_unlock(&srq->lock); 60 } 61 62 int mlx4_post_srq_recv(struct ibv_srq *ibsrq, 63 struct ibv_recv_wr *wr, 64 struct ibv_recv_wr **bad_wr) 65 { 66 struct mlx4_srq *srq = to_msrq(ibsrq); 67 struct mlx4_wqe_srq_next_seg *next; 68 struct mlx4_wqe_data_seg *scat; 69 int err = 0; 70 int nreq; 71 int i; 72 73 pthread_spin_lock(&srq->lock); 74 75 for (nreq = 0; wr; ++nreq, wr = wr->next) { 76 if (wr->num_sge > srq->max_gs) { 77 err = -1; 78 *bad_wr = wr; 79 break; 80 } 81 82 if (srq->head == srq->tail) { 83 /* SRQ is full*/ 84 err = -1; 85 *bad_wr = wr; 86 break; 87 } 88 89 srq->wrid[srq->head] = wr->wr_id; 90 91 next = get_wqe(srq, srq->head); 92 srq->head = be16toh(next->next_wqe_index); 93 scat = (struct mlx4_wqe_data_seg *) (next + 1); 94 95 for (i = 0; i < wr->num_sge; ++i) { 96 scat[i].byte_count = htobe32(wr->sg_list[i].length); 97 scat[i].lkey = htobe32(wr->sg_list[i].lkey); 98 scat[i].addr = htobe64(wr->sg_list[i].addr); 99 } 100 101 if (i < srq->max_gs) { 102 scat[i].byte_count = 0; 103 scat[i].lkey = htobe32(MLX4_INVALID_LKEY); 104 scat[i].addr = 0; 105 } 106 } 107 108 if (nreq) { 109 srq->counter += nreq; 110 111 /* 112 * Make sure that descriptors are written before 113 * we write doorbell record. 114 */ 115 udma_to_device_barrier(); 116 117 *srq->db = htobe32(srq->counter); 118 } 119 120 pthread_spin_unlock(&srq->lock); 121 122 return err; 123 } 124 125 int mlx4_alloc_srq_buf(struct ibv_pd *pd, struct ibv_srq_attr *attr, 126 struct mlx4_srq *srq) 127 { 128 struct mlx4_wqe_srq_next_seg *next; 129 struct mlx4_wqe_data_seg *scatter; 130 int size; 131 int buf_size; 132 int i; 133 134 srq->wrid = malloc(srq->max * sizeof (uint64_t)); 135 if (!srq->wrid) 136 return -1; 137 138 size = sizeof (struct mlx4_wqe_srq_next_seg) + 139 srq->max_gs * sizeof (struct mlx4_wqe_data_seg); 140 141 for (srq->wqe_shift = 5; 1 << srq->wqe_shift < size; ++srq->wqe_shift) 142 ; /* nothing */ 143 144 buf_size = srq->max << srq->wqe_shift; 145 146 if (mlx4_alloc_buf(&srq->buf, buf_size, 147 to_mdev(pd->context->device)->page_size)) { 148 free(srq->wrid); 149 return -1; 150 } 151 152 memset(srq->buf.buf, 0, buf_size); 153 154 /* 155 * Now initialize the SRQ buffer so that all of the WQEs are 156 * linked into the list of free WQEs. 157 */ 158 159 for (i = 0; i < srq->max; ++i) { 160 next = get_wqe(srq, i); 161 next->next_wqe_index = htobe16((i + 1) & (srq->max - 1)); 162 163 for (scatter = (void *) (next + 1); 164 (void *) scatter < (void *) next + (1 << srq->wqe_shift); 165 ++scatter) 166 scatter->lkey = htobe32(MLX4_INVALID_LKEY); 167 } 168 169 srq->head = 0; 170 srq->tail = srq->max - 1; 171 172 return 0; 173 } 174 175 void mlx4_init_xsrq_table(struct mlx4_xsrq_table *xsrq_table, int size) 176 { 177 memset(xsrq_table, 0, sizeof *xsrq_table); 178 xsrq_table->num_xsrq = size; 179 xsrq_table->shift = ffs(size) - 1 - MLX4_XSRQ_TABLE_BITS; 180 xsrq_table->mask = (1 << xsrq_table->shift) - 1; 181 182 pthread_mutex_init(&xsrq_table->mutex, NULL); 183 } 184 185 struct mlx4_srq *mlx4_find_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn) 186 { 187 int index; 188 189 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; 190 if (xsrq_table->xsrq_table[index].refcnt) 191 return xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask]; 192 193 return NULL; 194 } 195 196 int mlx4_store_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn, 197 struct mlx4_srq *srq) 198 { 199 int index, ret = 0; 200 201 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; 202 pthread_mutex_lock(&xsrq_table->mutex); 203 if (!xsrq_table->xsrq_table[index].refcnt) { 204 xsrq_table->xsrq_table[index].table = calloc(xsrq_table->mask + 1, 205 sizeof(struct mlx4_srq *)); 206 if (!xsrq_table->xsrq_table[index].table) { 207 ret = -1; 208 goto out; 209 } 210 } 211 212 xsrq_table->xsrq_table[index].refcnt++; 213 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = srq; 214 215 out: 216 pthread_mutex_unlock(&xsrq_table->mutex); 217 return ret; 218 } 219 220 void mlx4_clear_xsrq(struct mlx4_xsrq_table *xsrq_table, uint32_t srqn) 221 { 222 int index; 223 224 index = (srqn & (xsrq_table->num_xsrq - 1)) >> xsrq_table->shift; 225 pthread_mutex_lock(&xsrq_table->mutex); 226 227 if (--xsrq_table->xsrq_table[index].refcnt) 228 xsrq_table->xsrq_table[index].table[srqn & xsrq_table->mask] = NULL; 229 else 230 free(xsrq_table->xsrq_table[index].table); 231 232 pthread_mutex_unlock(&xsrq_table->mutex); 233 } 234 235 struct ibv_srq *mlx4_create_xrc_srq(struct ibv_context *context, 236 struct ibv_srq_init_attr_ex *attr_ex) 237 { 238 struct mlx4_create_xsrq cmd; 239 struct mlx4_create_srq_resp resp; 240 struct mlx4_srq *srq; 241 int ret; 242 243 /* Sanity check SRQ size before proceeding */ 244 if (attr_ex->attr.max_wr > 1 << 16 || attr_ex->attr.max_sge > 64) 245 return NULL; 246 247 srq = calloc(1, sizeof *srq); 248 if (!srq) 249 return NULL; 250 251 if (pthread_spin_init(&srq->lock, PTHREAD_PROCESS_PRIVATE)) 252 goto err; 253 254 srq->max = align_queue_size(attr_ex->attr.max_wr + 1); 255 srq->max_gs = attr_ex->attr.max_sge; 256 srq->counter = 0; 257 srq->ext_srq = 1; 258 259 if (mlx4_alloc_srq_buf(attr_ex->pd, &attr_ex->attr, srq)) 260 goto err; 261 262 srq->db = mlx4_alloc_db(to_mctx(context), MLX4_DB_TYPE_RQ); 263 if (!srq->db) 264 goto err_free; 265 266 *srq->db = 0; 267 268 cmd.buf_addr = (uintptr_t) srq->buf.buf; 269 cmd.db_addr = (uintptr_t) srq->db; 270 271 ret = ibv_cmd_create_srq_ex(context, &srq->verbs_srq, 272 sizeof(srq->verbs_srq), 273 attr_ex, 274 &cmd.ibv_cmd, sizeof cmd, 275 &resp.ibv_resp, sizeof resp); 276 if (ret) 277 goto err_db; 278 279 ret = mlx4_store_xsrq(&to_mctx(context)->xsrq_table, 280 srq->verbs_srq.srq_num, srq); 281 if (ret) 282 goto err_destroy; 283 284 return &srq->verbs_srq.srq; 285 286 err_destroy: 287 ibv_cmd_destroy_srq(&srq->verbs_srq.srq); 288 err_db: 289 mlx4_free_db(to_mctx(context), MLX4_DB_TYPE_RQ, srq->db); 290 err_free: 291 free(srq->wrid); 292 mlx4_free_buf(&srq->buf); 293 err: 294 free(srq); 295 return NULL; 296 } 297 298 int mlx4_destroy_xrc_srq(struct ibv_srq *srq) 299 { 300 struct mlx4_context *mctx = to_mctx(srq->context); 301 struct mlx4_srq *msrq = to_msrq(srq); 302 struct mlx4_cq *mcq; 303 int ret; 304 305 mcq = to_mcq(msrq->verbs_srq.cq); 306 mlx4_cq_clean(mcq, 0, msrq); 307 pthread_spin_lock(&mcq->lock); 308 mlx4_clear_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num); 309 pthread_spin_unlock(&mcq->lock); 310 311 ret = ibv_cmd_destroy_srq(srq); 312 if (ret) { 313 pthread_spin_lock(&mcq->lock); 314 mlx4_store_xsrq(&mctx->xsrq_table, msrq->verbs_srq.srq_num, msrq); 315 pthread_spin_unlock(&mcq->lock); 316 return ret; 317 } 318 319 mlx4_free_db(mctx, MLX4_DB_TYPE_RQ, msrq->db); 320 mlx4_free_buf(&msrq->buf); 321 free(msrq->wrid); 322 free(msrq); 323 324 return 0; 325 } 326