1 /*-
2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <dev/mlx5/mlx5_fpga/xfer.h>
34 #include <dev/mlx5/mlx5_fpga/conn.h>
35
36 struct xfer_state {
37 const struct mlx5_fpga_transaction *xfer;
38 /* Total transactions */
39 unsigned int start_count;
40 unsigned int done_count;
41 unsigned int error_count;
42 u8 status;
43 /* Inflight transactions */
44 unsigned int budget;
45 unsigned int inflight_count;
46 /* Chunking state */
47 size_t pos;
48 spinlock_t lock; /* Protects all members of this struct */
49 };
50
51 struct xfer_transaction {
52 struct xfer_state *xfer_state;
53 struct mlx5_fpga_transaction transaction;
54 };
55
56 static void trans_complete(const struct mlx5_fpga_transaction *complete,
57 u8 status);
58
xfer_complete(struct xfer_state * xfer_state)59 static void xfer_complete(struct xfer_state *xfer_state)
60 {
61 const struct mlx5_fpga_transaction *xfer = xfer_state->xfer;
62 u8 status = xfer_state->status;
63
64 kfree(xfer_state);
65 xfer->complete1(xfer, status);
66 }
67
68 /* Xfer state spin lock must be locked */
exec_more(struct xfer_state * xfer_state)69 static int exec_more(struct xfer_state *xfer_state)
70 {
71 struct xfer_transaction *xfer_trans;
72 size_t left, cur_size, page_size;
73 u64 pos_addr, ddr_base;
74 u8 *pos_data;
75 int ret = 0;
76
77 ddr_base = mlx5_fpga_ddr_base_get(xfer_state->xfer->conn->fdev);
78 page_size = (xfer_state->xfer->addr + xfer_state->pos < ddr_base) ?
79 sizeof(u32) : (1 << MLX5_FPGA_TRANSACTION_SEND_PAGE_BITS);
80
81 do {
82 if (xfer_state->status != IB_WC_SUCCESS) {
83 ret = -EIO;
84 break;
85 }
86
87 left = xfer_state->xfer->size - xfer_state->pos;
88 if (!left)
89 break;
90
91 xfer_trans = kzalloc(sizeof(*xfer_trans), GFP_ATOMIC);
92 if (!xfer_trans) {
93 ret = -ENOMEM;
94 break;
95 }
96
97 pos_addr = xfer_state->xfer->addr + xfer_state->pos;
98 pos_data = xfer_state->xfer->data + xfer_state->pos;
99
100 /* Determine largest possible transaction at this point */
101 cur_size = page_size - (pos_addr & (page_size - 1));
102 if (cur_size > MLX5_FPGA_TRANSACTION_MAX_SIZE)
103 cur_size = MLX5_FPGA_TRANSACTION_MAX_SIZE;
104 if (cur_size > left)
105 cur_size = left;
106
107 xfer_trans->xfer_state = xfer_state;
108 xfer_trans->transaction.addr = pos_addr;
109 xfer_trans->transaction.complete1 = trans_complete;
110 xfer_trans->transaction.conn = xfer_state->xfer->conn;
111 xfer_trans->transaction.data = pos_data;
112 xfer_trans->transaction.direction = xfer_state->xfer->direction;
113 xfer_trans->transaction.size = cur_size;
114
115 xfer_state->start_count++;
116 xfer_state->inflight_count++;
117 mlx5_fpga_dbg(xfer_state->xfer->conn->fdev, "Starting %zu bytes at %p done; %u started %u inflight %u done %u error\n",
118 xfer_trans->transaction.size,
119 xfer_trans->transaction.data,
120 xfer_state->start_count,
121 xfer_state->inflight_count,
122 xfer_state->done_count,
123 xfer_state->error_count);
124 ret = mlx5_fpga_trans_exec(&xfer_trans->transaction);
125 if (ret) {
126 xfer_state->start_count--;
127 xfer_state->inflight_count--;
128 if (ret == -EBUSY)
129 ret = 0;
130
131 if (ret) {
132 mlx5_fpga_warn(xfer_state->xfer->conn->fdev, "Transfer failed to start transaction: %d. %u started %u done %u error\n",
133 ret, xfer_state->start_count,
134 xfer_state->done_count,
135 xfer_state->error_count);
136 xfer_state->status = IB_WC_GENERAL_ERR;
137 }
138 kfree(xfer_trans);
139 break;
140 }
141 xfer_state->pos += cur_size;
142 if (xfer_state->inflight_count >= xfer_state->budget)
143 break;
144 } while (cur_size != left);
145
146 return ret;
147 }
148
trans_complete(const struct mlx5_fpga_transaction * complete,u8 status)149 static void trans_complete(const struct mlx5_fpga_transaction *complete,
150 u8 status)
151 {
152 struct xfer_transaction *xfer_trans;
153 struct xfer_state *xfer_state;
154 unsigned long flags;
155 bool done = false;
156 int ret;
157
158 xfer_trans = container_of(complete, struct xfer_transaction,
159 transaction);
160 xfer_state = xfer_trans->xfer_state;
161 mlx5_fpga_dbg(complete->conn->fdev, "Transaction %zu bytes at %p done, status %u; %u started %u inflight %u done %u error\n",
162 xfer_trans->transaction.size,
163 xfer_trans->transaction.data, status,
164 xfer_state->start_count, xfer_state->inflight_count,
165 xfer_state->done_count, xfer_state->error_count);
166 kfree(xfer_trans);
167
168 spin_lock_irqsave(&xfer_state->lock, flags);
169
170 if (status != IB_WC_SUCCESS) {
171 xfer_state->error_count++;
172 mlx5_fpga_warn(complete->conn->fdev, "Transaction failed during transfer. %u started %u inflight %u done %u error\n",
173 xfer_state->start_count,
174 xfer_state->inflight_count,
175 xfer_state->done_count, xfer_state->error_count);
176 if (xfer_state->status == IB_WC_SUCCESS)
177 xfer_state->status = status;
178 } else {
179 xfer_state->done_count++;
180 }
181 ret = exec_more(xfer_state);
182
183 xfer_state->inflight_count--;
184 if (!xfer_state->inflight_count)
185 done = true;
186
187 spin_unlock_irqrestore(&xfer_state->lock, flags);
188
189 if (done)
190 xfer_complete(xfer_state);
191 }
192
mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction * xfer)193 int mlx5_fpga_xfer_exec(const struct mlx5_fpga_transaction *xfer)
194 {
195 u64 base = mlx5_fpga_ddr_base_get(xfer->conn->fdev);
196 u64 size = mlx5_fpga_ddr_size_get(xfer->conn->fdev);
197 struct xfer_state *xfer_state;
198 unsigned long flags;
199 bool done = false;
200 int ret = 0;
201
202 if (xfer->addr + xfer->size > base + size) {
203 mlx5_fpga_warn(xfer->conn->fdev, "Transfer ends at %jx outside of DDR range %jx\n",
204 (uintmax_t)(xfer->addr + xfer->size), (uintmax_t)(base + size));
205 return -EINVAL;
206 }
207
208 if (xfer->addr & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) {
209 mlx5_fpga_warn(xfer->conn->fdev, "Transfer address %jx not aligned\n",
210 (uintmax_t)xfer->addr);
211 return -EINVAL;
212 }
213
214 if (xfer->size & MLX5_FPGA_TRANSACTION_SEND_ALIGN_BITS) {
215 mlx5_fpga_warn(xfer->conn->fdev, "Transfer size %zu not aligned\n",
216 xfer->size);
217 return -EINVAL;
218 }
219
220 if (xfer->size < 1) {
221 mlx5_fpga_warn(xfer->conn->fdev, "Empty transfer size %zu not allowed\n",
222 xfer->size);
223 return -EINVAL;
224 }
225
226 xfer_state = kzalloc(sizeof(*xfer_state), GFP_KERNEL);
227 xfer_state->xfer = xfer;
228 xfer_state->status = IB_WC_SUCCESS;
229 xfer_state->budget = 7;
230 spin_lock_init(&xfer_state->lock);
231 spin_lock_irqsave(&xfer_state->lock, flags);
232
233 ret = exec_more(xfer_state);
234 if (ret && (xfer_state->start_count == 0))
235 done = true;
236
237 spin_unlock_irqrestore(&xfer_state->lock, flags);
238
239 if (done)
240 xfer_complete(xfer_state);
241 return ret;
242 }
243