1 /*-
2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <dev/mlx5/driver.h>
34
35 #include <dev/mlx5/mlx5_core/mlx5_core.h>
36 #include <dev/mlx5/mlx5_fpga/ipsec.h>
37 #include <dev/mlx5/mlx5_fpga/sdk.h>
38 #include <dev/mlx5/mlx5_fpga/core.h>
39
40 #define SBU_QP_QUEUE_SIZE 8
41
42 enum mlx5_ipsec_response_syndrome {
43 MLX5_IPSEC_RESPONSE_SUCCESS = 0,
44 MLX5_IPSEC_RESPONSE_ILLEGAL_REQUEST = 1,
45 MLX5_IPSEC_RESPONSE_SADB_ISSUE = 2,
46 MLX5_IPSEC_RESPONSE_WRITE_RESPONSE_ISSUE = 3,
47 };
48
49 enum mlx5_fpga_ipsec_sacmd_status {
50 MLX5_FPGA_IPSEC_SACMD_PENDING,
51 MLX5_FPGA_IPSEC_SACMD_SEND_FAIL,
52 MLX5_FPGA_IPSEC_SACMD_COMPLETE,
53 };
54
55 struct mlx5_ipsec_command_context {
56 struct mlx5_fpga_dma_buf buf;
57 struct mlx5_accel_ipsec_sa sa;
58 enum mlx5_fpga_ipsec_sacmd_status status;
59 int status_code;
60 struct completion complete;
61 struct mlx5_fpga_device *dev;
62 struct list_head list; /* Item in pending_cmds */
63 };
64
65 struct mlx5_ipsec_sadb_resp {
66 __be32 syndrome;
67 __be32 sw_sa_handle;
68 u8 reserved[24];
69 } __packed;
70
71 struct mlx5_fpga_ipsec {
72 struct list_head pending_cmds;
73 spinlock_t pending_cmds_lock; /* Protects pending_cmds */
74 u32 caps[MLX5_ST_SZ_DW(ipsec_extended_cap)];
75 struct mlx5_fpga_conn *conn;
76 };
77
mlx5_fpga_is_ipsec_device(struct mlx5_core_dev * mdev)78 static bool mlx5_fpga_is_ipsec_device(struct mlx5_core_dev *mdev)
79 {
80 if (!mdev->fpga || !MLX5_CAP_GEN(mdev, fpga))
81 return false;
82
83 if (MLX5_CAP_FPGA(mdev, ieee_vendor_id) !=
84 MLX5_FPGA_CAP_SANDBOX_VENDOR_ID_MLNX)
85 return false;
86
87 if (MLX5_CAP_FPGA(mdev, sandbox_product_id) !=
88 MLX5_FPGA_CAP_SANDBOX_PRODUCT_ID_IPSEC)
89 return false;
90
91 return true;
92 }
93
mlx5_fpga_ipsec_send_complete(struct mlx5_fpga_conn * conn,struct mlx5_fpga_device * fdev,struct mlx5_fpga_dma_buf * buf,u8 status)94 static void mlx5_fpga_ipsec_send_complete(struct mlx5_fpga_conn *conn,
95 struct mlx5_fpga_device *fdev,
96 struct mlx5_fpga_dma_buf *buf,
97 u8 status)
98 {
99 struct mlx5_ipsec_command_context *context;
100
101 if (status) {
102 context = container_of(buf, struct mlx5_ipsec_command_context,
103 buf);
104 mlx5_fpga_warn(fdev, "IPSec command send failed with status %u\n",
105 status);
106 context->status = MLX5_FPGA_IPSEC_SACMD_SEND_FAIL;
107 complete(&context->complete);
108 }
109 }
110
syndrome_to_errno(enum mlx5_ipsec_response_syndrome syndrome)111 static inline int syndrome_to_errno(enum mlx5_ipsec_response_syndrome syndrome)
112 {
113 switch (syndrome) {
114 case MLX5_IPSEC_RESPONSE_SUCCESS:
115 return 0;
116 case MLX5_IPSEC_RESPONSE_SADB_ISSUE:
117 return -EEXIST;
118 case MLX5_IPSEC_RESPONSE_ILLEGAL_REQUEST:
119 return -EINVAL;
120 case MLX5_IPSEC_RESPONSE_WRITE_RESPONSE_ISSUE:
121 return -EIO;
122 }
123 return -EIO;
124 }
125
mlx5_fpga_ipsec_recv(void * cb_arg,struct mlx5_fpga_dma_buf * buf)126 static void mlx5_fpga_ipsec_recv(void *cb_arg, struct mlx5_fpga_dma_buf *buf)
127 {
128 struct mlx5_ipsec_sadb_resp *resp = buf->sg[0].data;
129 struct mlx5_ipsec_command_context *context;
130 enum mlx5_ipsec_response_syndrome syndrome;
131 struct mlx5_fpga_device *fdev = cb_arg;
132 unsigned long flags;
133
134 if (buf->sg[0].size < sizeof(*resp)) {
135 mlx5_fpga_warn(fdev, "Short receive from FPGA IPSec: %u < %zu bytes\n",
136 buf->sg[0].size, sizeof(*resp));
137 return;
138 }
139
140 mlx5_fpga_dbg(fdev, "mlx5_ipsec recv_cb syndrome %08x sa_id %x\n",
141 ntohl(resp->syndrome), ntohl(resp->sw_sa_handle));
142
143 spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags);
144 context = list_first_entry_or_null(&fdev->ipsec->pending_cmds,
145 struct mlx5_ipsec_command_context,
146 list);
147 if (context)
148 list_del(&context->list);
149 spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags);
150
151 if (!context) {
152 mlx5_fpga_warn(fdev, "Received IPSec offload response without pending command request\n");
153 return;
154 }
155 mlx5_fpga_dbg(fdev, "Handling response for %p\n", context);
156
157 if (context->sa.sw_sa_handle != resp->sw_sa_handle) {
158 mlx5_fpga_err(fdev, "mismatch SA handle. cmd 0x%08x vs resp 0x%08x\n",
159 ntohl(context->sa.sw_sa_handle),
160 ntohl(resp->sw_sa_handle));
161 return;
162 }
163
164 syndrome = ntohl(resp->syndrome);
165 context->status_code = syndrome_to_errno(syndrome);
166 context->status = MLX5_FPGA_IPSEC_SACMD_COMPLETE;
167
168 if (context->status_code)
169 mlx5_fpga_warn(fdev, "IPSec SADB command failed with syndrome %08x\n",
170 syndrome);
171 complete(&context->complete);
172 }
173
mlx5_fpga_ipsec_sa_cmd_exec(struct mlx5_core_dev * mdev,struct mlx5_accel_ipsec_sa * cmd)174 void *mlx5_fpga_ipsec_sa_cmd_exec(struct mlx5_core_dev *mdev,
175 struct mlx5_accel_ipsec_sa *cmd)
176 {
177 struct mlx5_ipsec_command_context *context;
178 struct mlx5_fpga_device *fdev = mdev->fpga;
179 unsigned long flags;
180 int res = 0;
181
182 BUILD_BUG_ON((sizeof(struct mlx5_accel_ipsec_sa) & 3) != 0);
183 if (!fdev || !fdev->ipsec)
184 return ERR_PTR(-EOPNOTSUPP);
185
186 context = kzalloc(sizeof(*context), GFP_ATOMIC);
187 if (!context)
188 return ERR_PTR(-ENOMEM);
189
190 memcpy(&context->sa, cmd, sizeof(*cmd));
191 context->buf.complete = mlx5_fpga_ipsec_send_complete;
192 context->buf.sg[0].size = sizeof(context->sa);
193 context->buf.sg[0].data = &context->sa;
194 init_completion(&context->complete);
195 context->dev = fdev;
196 spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags);
197 list_add_tail(&context->list, &fdev->ipsec->pending_cmds);
198 spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags);
199
200 context->status = MLX5_FPGA_IPSEC_SACMD_PENDING;
201
202 res = mlx5_fpga_sbu_conn_sendmsg(fdev->ipsec->conn, &context->buf);
203 if (res) {
204 mlx5_fpga_warn(fdev, "Failure sending IPSec command: %d\n",
205 res);
206 spin_lock_irqsave(&fdev->ipsec->pending_cmds_lock, flags);
207 list_del(&context->list);
208 spin_unlock_irqrestore(&fdev->ipsec->pending_cmds_lock, flags);
209 kfree(context);
210 return ERR_PTR(res);
211 }
212 /* Context will be freed by wait func after completion */
213 return context;
214 }
215
mlx5_fpga_ipsec_sa_cmd_wait(void * ctx)216 int mlx5_fpga_ipsec_sa_cmd_wait(void *ctx)
217 {
218 struct mlx5_ipsec_command_context *context = ctx;
219 int res;
220
221 res = wait_for_completion/*_killable XXXKIB*/(&context->complete);
222 if (res) {
223 mlx5_fpga_warn(context->dev, "Failure waiting for IPSec command response\n");
224 return -EINTR;
225 }
226
227 if (context->status == MLX5_FPGA_IPSEC_SACMD_COMPLETE)
228 res = context->status_code;
229 else
230 res = -EIO;
231
232 kfree(context);
233 return res;
234 }
235
mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev * mdev)236 u32 mlx5_fpga_ipsec_device_caps(struct mlx5_core_dev *mdev)
237 {
238 struct mlx5_fpga_device *fdev = mdev->fpga;
239 u32 ret = 0;
240
241 if (mlx5_fpga_is_ipsec_device(mdev))
242 ret |= MLX5_ACCEL_IPSEC_DEVICE;
243 else
244 return ret;
245
246 if (!fdev->ipsec)
247 return ret;
248
249 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, esp))
250 ret |= MLX5_ACCEL_IPSEC_ESP;
251
252 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, ipv6))
253 ret |= MLX5_ACCEL_IPSEC_IPV6;
254
255 if (MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps, lso))
256 ret |= MLX5_ACCEL_IPSEC_LSO;
257
258 return ret;
259 }
260
mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev * mdev)261 unsigned int mlx5_fpga_ipsec_counters_count(struct mlx5_core_dev *mdev)
262 {
263 struct mlx5_fpga_device *fdev = mdev->fpga;
264
265 if (!fdev || !fdev->ipsec)
266 return 0;
267
268 return MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps,
269 number_of_ipsec_counters);
270 }
271
mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev * mdev,u64 * counters,unsigned int counters_count)272 int mlx5_fpga_ipsec_counters_read(struct mlx5_core_dev *mdev, u64 *counters,
273 unsigned int counters_count)
274 {
275 struct mlx5_fpga_device *fdev = mdev->fpga;
276 unsigned int i;
277 __be32 *data;
278 u32 count;
279 u64 addr;
280 int ret;
281
282 if (!fdev || !fdev->ipsec)
283 return 0;
284
285 addr = (u64)MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps,
286 ipsec_counters_addr_low) +
287 ((u64)MLX5_GET(ipsec_extended_cap, fdev->ipsec->caps,
288 ipsec_counters_addr_high) << 32);
289
290 count = mlx5_fpga_ipsec_counters_count(mdev);
291
292 data = kzalloc(sizeof(*data) * count * 2, GFP_KERNEL);
293 if (!data) {
294 ret = -ENOMEM;
295 goto out;
296 }
297
298 ret = mlx5_fpga_mem_read(fdev, count * sizeof(u64), addr, data,
299 MLX5_FPGA_ACCESS_TYPE_DONTCARE);
300 if (ret < 0) {
301 mlx5_fpga_err(fdev, "Failed to read IPSec counters from HW: %d\n",
302 ret);
303 goto out;
304 }
305 ret = 0;
306
307 if (count > counters_count)
308 count = counters_count;
309
310 /* Each counter is low word, then high. But each word is big-endian */
311 for (i = 0; i < count; i++)
312 counters[i] = (u64)ntohl(data[i * 2]) |
313 ((u64)ntohl(data[i * 2 + 1]) << 32);
314
315 out:
316 kfree(data);
317 return ret;
318 }
319
mlx5_fpga_ipsec_init(struct mlx5_core_dev * mdev)320 int mlx5_fpga_ipsec_init(struct mlx5_core_dev *mdev)
321 {
322 struct mlx5_fpga_conn_attr init_attr = {0};
323 struct mlx5_fpga_device *fdev = mdev->fpga;
324 struct mlx5_fpga_conn *conn;
325 int err;
326
327 if (!mlx5_fpga_is_ipsec_device(mdev))
328 return 0;
329
330 fdev->ipsec = kzalloc(sizeof(*fdev->ipsec), GFP_KERNEL);
331 if (!fdev->ipsec)
332 return -ENOMEM;
333
334 err = mlx5_fpga_get_sbu_caps(fdev, sizeof(fdev->ipsec->caps),
335 fdev->ipsec->caps);
336 if (err) {
337 mlx5_fpga_err(fdev, "Failed to retrieve IPSec extended capabilities: %d\n",
338 err);
339 goto error;
340 }
341
342 INIT_LIST_HEAD(&fdev->ipsec->pending_cmds);
343 spin_lock_init(&fdev->ipsec->pending_cmds_lock);
344
345 init_attr.rx_size = SBU_QP_QUEUE_SIZE;
346 init_attr.tx_size = SBU_QP_QUEUE_SIZE;
347 init_attr.recv_cb = mlx5_fpga_ipsec_recv;
348 init_attr.cb_arg = fdev;
349 conn = mlx5_fpga_sbu_conn_create(fdev, &init_attr);
350 if (IS_ERR(conn)) {
351 err = PTR_ERR(conn);
352 mlx5_fpga_err(fdev, "Error creating IPSec command connection %d\n",
353 err);
354 goto error;
355 }
356 fdev->ipsec->conn = conn;
357 return 0;
358
359 error:
360 kfree(fdev->ipsec);
361 fdev->ipsec = NULL;
362 return err;
363 }
364
mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev * mdev)365 void mlx5_fpga_ipsec_cleanup(struct mlx5_core_dev *mdev)
366 {
367 struct mlx5_fpga_device *fdev = mdev->fpga;
368
369 if (!mlx5_fpga_is_ipsec_device(mdev))
370 return;
371
372 mlx5_fpga_sbu_conn_destroy(fdev->ipsec->conn);
373 kfree(fdev->ipsec);
374 fdev->ipsec = NULL;
375 }
376