1 /*- 2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 * $FreeBSD$ 33 */ 34 35 #include <dev/mlx5/mlx5_fpga/xfer.h> 36 #include <dev/mlx5/mlx5_fpga/conn.h> 37 38 struct xfer_state { 39 const struct mlx5_fpga_transaction *xfer; 40 /* Total transactions */ 41 unsigned int start_count; 42 unsigned int done_count; 43 unsigned int error_count; 44 u8 status; 45 /* Inflight transactions */ 46 unsigned int budget; 47 unsigned int inflight_count; 48 /* Chunking state */ 49 size_t pos; 50 spinlock_t lock; /* Protects all members of this struct */ 51 }; 52 53 struct xfer_transaction { 54 struct xfer_state *xfer_state; 55 struct mlx5_fpga_transaction transaction; 56 }; 57 58 static void trans_complete(const struct mlx5_fpga_transaction *complete, 59 u8 status); 60 61 static void xfer_complete(struct xfer_state *xfer_state) 62 { 63 const struct mlx5_fpga_transaction *xfer = xfer_state->xfer; 64 u8 status = xfer_state->status; 65 66 kfree(xfer_state); 67 xfer->complete1(xfer, status); 68 } 69 70 /* Xfer state spin lock must be locked */ 71 static int exec_more(struct xfer_state *xfer_state) 72 { 73 struct xfer_transaction *xfer_trans; 74 size_t left, cur_size, page_size; 75 u64 pos_addr, ddr_base; 76 u8 *pos_data; 77 int ret = 0; 78 79 ddr_base = mlx5_fpga_ddr_base_get(xfer_state->xfer->conn->fdev); 80 page_size = (xfer_state->xfer->addr + xfer_state->pos < ddr_base) ? 81 sizeof(u32) : (1 << MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS); 82 83 do { 84 if (xfer_state->status != IB_WC_SUCCESS) { 85 ret = -EIO; 86 break; 87 } 88 89 left = xfer_state->xfer->size - xfer_state->pos; 90 if (!left) 91 break; 92 93 xfer_trans = kzalloc(sizeof(*xfer_trans), GFP_ATOMIC); 94 if (!xfer_trans) { 95 ret = -ENOMEM; 96 break; 97 } 98 99 pos_addr = xfer_state->xfer->addr + xfer_state->pos; 100 pos_data = xfer_state->xfer->data + xfer_state->pos; 101 102 /* Determine largest possible transaction at this point */ 103 cur_size = page_size - (pos_addr & (page_size - 1)); 104 if (cur_size > MLX5_FPGA_TRANSACTION_MAX_SIZE) 105 cur_size = MLX5_FPGA_TRANSACTION_MAX_SIZE; 106 if (cur_size > left) 107 cur_size = left; 108 109 xfer_trans->xfer_state = xfer_state; 110 xfer_trans->transaction.addr = pos_addr; 111 xfer_trans->transaction.complete1 = trans_complete; 112 xfer_trans->transaction.conn = xfer_state->xfer->conn; 113 xfer_trans->transaction.data = pos_data; 114 xfer_trans->transaction.direction = xfer_state->xfer->direction; 115 xfer_trans->transaction.size = cur_size; 116 117 xfer_state->start_count++; 118 xfer_state->inflight_count++; 119 mlx5_fpga_dbg(xfer_state->xfer->conn->fdev, "Starting %zu bytes at %p done; %u started %u inflight %u done %u error\n", 120 xfer_trans->transaction.size, 121 xfer_trans->transaction.data, 122 xfer_state->start_count, 123 xfer_state->inflight_count, 124 xfer_state->done_count, 125 xfer_state->error_count); 126 ret = mlx5_fpga_trans_exec(&xfer_trans->transaction); 127 if (ret) { 128 xfer_state->start_count--; 129 xfer_state->inflight_count--; 130 if (ret == -EBUSY) 131 ret = 0; 132 133 if (ret) { 134 mlx5_fpga_warn(xfer_state->xfer->conn->fdev, "Transfer failed to start transaction: %d. %u started %u done %u error\n", 135 ret, xfer_state->start_count, 136 xfer_state->done_count, 137 xfer_state->error_count); 138 xfer_state->status = IB_WC_GENERAL_ERR; 139 } 140 kfree(xfer_trans); 141 break; 142 } 143 xfer_state->pos += cur_size; 144 if (xfer_state->inflight_count >= xfer_state->budget) 145 break; 146 } while (cur_size != left); 147 148 return ret; 149 } 150 151 static void trans_complete(const struct mlx5_fpga_transaction *complete, 152 u8 status) 153 { 154 struct xfer_transaction *xfer_trans; 155 struct xfer_state *xfer_state; 156 unsigned long flags; 157 bool done = false; 158 int ret; 159 160 xfer_trans = container_of(complete, struct xfer_transaction, 161 transaction); 162 xfer_state = xfer_trans->xfer_state; 163 mlx5_fpga_dbg(complete->conn->fdev, "Transaction %zu bytes at %p done, status %u; %u started %u inflight %u done %u error\n", 164 xfer_trans->transaction.size, 165 xfer_trans->transaction.data, status, 166 xfer_state->start_count, xfer_state->inflight_count, 167 xfer_state->done_count, xfer_state->error_count); 168 kfree(xfer_trans); 169 170 spin_lock_irqsave(&xfer_state->lock, flags); 171 172 if (status != IB_WC_SUCCESS) { 173 xfer_state->error_count++; 174 mlx5_fpga_warn(complete->conn->fdev, "Transaction failed during transfer. %u started %u inflight %u done %u error\n", 175 xfer_state->start_count, 176 xfer_state->inflight_count, 177 xfer_state->done_count, xfer_state->error_count); 178 if (xfer_state->status == IB_WC_SUCCESS) 179 xfer_state->status = status; 180 } else { 181 xfer_state->done_count++; 182 } 183 ret = exec_more(xfer_state); 184 185 xfer_state->inflight_count--; 186 if (!xfer_state->inflight_count) 187 done = true; 188 189 spin_unlock_irqrestore(&xfer_state->lock, flags); 190 191 if (done) 192 xfer_complete(xfer_state); 193 } 194 195 int mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer) 196 { 197 u64 base = mlx5_fpga_ddr_base_get(xfer->conn->fdev); 198 u64 size = mlx5_fpga_ddr_size_get(xfer->conn->fdev); 199 struct xfer_state *xfer_state; 200 unsigned long flags; 201 bool done = false; 202 int ret = 0; 203 204 if (xfer->addr + xfer->size > base + size) { 205 mlx5_fpga_warn(xfer->conn->fdev, "Transfer ends at %jx outside of DDR range %jx\n", 206 (uintmax_t)(xfer->addr + xfer->size), (uintmax_t)(base + size)); 207 return -EINVAL; 208 } 209 210 if (xfer->addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { 211 mlx5_fpga_warn(xfer->conn->fdev, "Transfer address %jx not aligned\n", 212 (uintmax_t)xfer->addr); 213 return -EINVAL; 214 } 215 216 if (xfer->size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) { 217 mlx5_fpga_warn(xfer->conn->fdev, "Transfer size %zu not aligned\n", 218 xfer->size); 219 return -EINVAL; 220 } 221 222 if (xfer->size < 1) { 223 mlx5_fpga_warn(xfer->conn->fdev, "Empty transfer size %zu not allowed\n", 224 xfer->size); 225 return -EINVAL; 226 } 227 228 xfer_state = kzalloc(sizeof(*xfer_state), GFP_KERNEL); 229 xfer_state->xfer = xfer; 230 xfer_state->status = IB_WC_SUCCESS; 231 xfer_state->budget = 7; 232 spin_lock_init(&xfer_state->lock); 233 spin_lock_irqsave(&xfer_state->lock, flags); 234 235 ret = exec_more(xfer_state); 236 if (ret && (xfer_state->start_count == 0)) 237 done = true; 238 239 spin_unlock_irqrestore(&xfer_state->lock, flags); 240 241 if (done) 242 xfer_complete(xfer_state); 243 return ret; 244 } 245