1 /* 2 * Copyright (c) 2012 Mellanox Technologies, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <config.h> 34 35 #include <stdlib.h> 36 #include <pthread.h> 37 #include <string.h> 38 #include <errno.h> 39 40 #include "mlx5.h" 41 #include "doorbell.h" 42 #include "wqe.h" 43 44 static void *get_wqe(struct mlx5_srq *srq, int n) 45 { 46 return srq->buf.buf + (n << srq->wqe_shift); 47 } 48 49 int mlx5_copy_to_recv_srq(struct mlx5_srq *srq, int idx, void *buf, int size) 50 { 51 struct mlx5_wqe_srq_next_seg *next; 52 struct mlx5_wqe_data_seg *scat; 53 int copy; 54 int i; 55 int max = 1 << (srq->wqe_shift - 4); 56 57 next = get_wqe(srq, idx); 58 scat = (struct mlx5_wqe_data_seg *) (next + 1); 59 60 for (i = 0; i < max; ++i) { 61 copy = min_t(long, size, be32toh(scat->byte_count)); 62 memcpy((void *)(unsigned long)be64toh(scat->addr), buf, copy); 63 size -= copy; 64 if (size <= 0) 65 return IBV_WC_SUCCESS; 66 67 buf += copy; 68 ++scat; 69 } 70 return IBV_WC_LOC_LEN_ERR; 71 } 72 73 void mlx5_free_srq_wqe(struct mlx5_srq *srq, int ind) 74 { 75 struct mlx5_wqe_srq_next_seg *next; 76 77 mlx5_spin_lock(&srq->lock); 78 79 next = get_wqe(srq, srq->tail); 80 next->next_wqe_index = htobe16(ind); 81 srq->tail = ind; 82 83 mlx5_spin_unlock(&srq->lock); 84 } 85 86 int mlx5_post_srq_recv(struct ibv_srq *ibsrq, 87 struct ibv_recv_wr *wr, 88 struct ibv_recv_wr **bad_wr) 89 { 90 struct mlx5_srq *srq = to_msrq(ibsrq); 91 struct mlx5_wqe_srq_next_seg *next; 92 struct mlx5_wqe_data_seg *scat; 93 int err = 0; 94 int nreq; 95 int i; 96 97 mlx5_spin_lock(&srq->lock); 98 99 for (nreq = 0; wr; ++nreq, wr = wr->next) { 100 if (wr->num_sge > srq->max_gs) { 101 err = EINVAL; 102 *bad_wr = wr; 103 break; 104 } 105 106 if (srq->head == srq->tail) { 107 /* SRQ is full*/ 108 err = ENOMEM; 109 *bad_wr = wr; 110 break; 111 } 112 113 srq->wrid[srq->head] = wr->wr_id; 114 115 next = get_wqe(srq, srq->head); 116 srq->head = be16toh(next->next_wqe_index); 117 scat = (struct mlx5_wqe_data_seg *) (next + 1); 118 119 for (i = 0; i < wr->num_sge; ++i) { 120 scat[i].byte_count = htobe32(wr->sg_list[i].length); 121 scat[i].lkey = htobe32(wr->sg_list[i].lkey); 122 scat[i].addr = htobe64(wr->sg_list[i].addr); 123 } 124 125 if (i < srq->max_gs) { 126 scat[i].byte_count = 0; 127 scat[i].lkey = htobe32(MLX5_INVALID_LKEY); 128 scat[i].addr = 0; 129 } 130 } 131 132 if (nreq) { 133 srq->counter += nreq; 134 135 /* 136 * Make sure that descriptors are written before 137 * we write doorbell record. 138 */ 139 udma_to_device_barrier(); 140 141 *srq->db = htobe32(srq->counter); 142 } 143 144 mlx5_spin_unlock(&srq->lock); 145 146 return err; 147 } 148 149 int mlx5_alloc_srq_buf(struct ibv_context *context, struct mlx5_srq *srq) 150 { 151 struct mlx5_wqe_srq_next_seg *next; 152 int size; 153 int buf_size; 154 int i; 155 struct mlx5_context *ctx; 156 157 ctx = to_mctx(context); 158 159 if (srq->max_gs < 0) { 160 errno = EINVAL; 161 return -1; 162 } 163 164 srq->wrid = malloc(srq->max * sizeof *srq->wrid); 165 if (!srq->wrid) 166 return -1; 167 168 size = sizeof(struct mlx5_wqe_srq_next_seg) + 169 srq->max_gs * sizeof(struct mlx5_wqe_data_seg); 170 size = max(32, size); 171 172 size = mlx5_round_up_power_of_two(size); 173 174 if (size > ctx->max_recv_wr) { 175 errno = EINVAL; 176 return -1; 177 } 178 srq->max_gs = (size - sizeof(struct mlx5_wqe_srq_next_seg)) / 179 sizeof(struct mlx5_wqe_data_seg); 180 181 srq->wqe_shift = mlx5_ilog2(size); 182 183 buf_size = srq->max * size; 184 185 if (mlx5_alloc_buf(&srq->buf, buf_size, 186 to_mdev(context->device)->page_size)) { 187 free(srq->wrid); 188 return -1; 189 } 190 191 memset(srq->buf.buf, 0, buf_size); 192 193 /* 194 * Now initialize the SRQ buffer so that all of the WQEs are 195 * linked into the list of free WQEs. 196 */ 197 198 for (i = 0; i < srq->max; ++i) { 199 next = get_wqe(srq, i); 200 next->next_wqe_index = htobe16((i + 1) & (srq->max - 1)); 201 } 202 203 srq->head = 0; 204 srq->tail = srq->max - 1; 205 206 return 0; 207 } 208 209 struct mlx5_srq *mlx5_find_srq(struct mlx5_context *ctx, uint32_t srqn) 210 { 211 int tind = srqn >> MLX5_SRQ_TABLE_SHIFT; 212 213 if (ctx->srq_table[tind].refcnt) 214 return ctx->srq_table[tind].table[srqn & MLX5_SRQ_TABLE_MASK]; 215 else 216 return NULL; 217 } 218 219 int mlx5_store_srq(struct mlx5_context *ctx, uint32_t srqn, 220 struct mlx5_srq *srq) 221 { 222 int tind = srqn >> MLX5_SRQ_TABLE_SHIFT; 223 224 if (!ctx->srq_table[tind].refcnt) { 225 ctx->srq_table[tind].table = calloc(MLX5_QP_TABLE_MASK + 1, 226 sizeof(struct mlx5_qp *)); 227 if (!ctx->srq_table[tind].table) 228 return -1; 229 } 230 231 ++ctx->srq_table[tind].refcnt; 232 ctx->srq_table[tind].table[srqn & MLX5_QP_TABLE_MASK] = srq; 233 return 0; 234 } 235 236 void mlx5_clear_srq(struct mlx5_context *ctx, uint32_t srqn) 237 { 238 int tind = srqn >> MLX5_QP_TABLE_SHIFT; 239 240 if (!--ctx->srq_table[tind].refcnt) 241 free(ctx->srq_table[tind].table); 242 else 243 ctx->srq_table[tind].table[srqn & MLX5_SRQ_TABLE_MASK] = NULL; 244 } 245