1 /*-
2 * Copyright (c) 2017 Mellanox Technologies. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32
33 #include <linux/etherdevice.h>
34 #include <dev/mlx5/vport.h>
35 #include <dev/mlx5/mlx5_core/mlx5_core.h>
36 #include <dev/mlx5/mlx5_lib/mlx5.h>
37 #include <dev/mlx5/mlx5_fpga/core.h>
38 #include <dev/mlx5/mlx5_fpga/conn.h>
39
40 #define MLX5_FPGA_PKEY 0xFFFF
41 #define MLX5_FPGA_PKEY_INDEX 0 /* RoCE PKEY 0xFFFF is always at index 0 */
42 #define MLX5_FPGA_RECV_SIZE 2048
43 #define MLX5_FPGA_PORT_NUM 1
44 #define MLX5_FPGA_CQ_BUDGET 64
45
mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)46 static int mlx5_fpga_conn_map_buf(struct mlx5_fpga_conn *conn,
47 struct mlx5_fpga_dma_buf *buf)
48 {
49 struct device *dma_device;
50 int err = 0;
51
52 if (unlikely(!buf->sg[0].data))
53 goto out;
54
55 dma_device = &conn->fdev->mdev->pdev->dev;
56 buf->sg[0].dma_addr = dma_map_single(dma_device, buf->sg[0].data,
57 buf->sg[0].size, buf->dma_dir);
58 err = dma_mapping_error(dma_device, buf->sg[0].dma_addr);
59 if (unlikely(err)) {
60 mlx5_fpga_warn(conn->fdev, "DMA error on sg 0: %d\n", err);
61 err = -ENOMEM;
62 goto out;
63 }
64
65 if (!buf->sg[1].data)
66 goto out;
67
68 buf->sg[1].dma_addr = dma_map_single(dma_device, buf->sg[1].data,
69 buf->sg[1].size, buf->dma_dir);
70 err = dma_mapping_error(dma_device, buf->sg[1].dma_addr);
71 if (unlikely(err)) {
72 mlx5_fpga_warn(conn->fdev, "DMA error on sg 1: %d\n", err);
73 dma_unmap_single(dma_device, buf->sg[0].dma_addr,
74 buf->sg[0].size, buf->dma_dir);
75 err = -ENOMEM;
76 }
77
78 out:
79 return err;
80 }
81
mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)82 static void mlx5_fpga_conn_unmap_buf(struct mlx5_fpga_conn *conn,
83 struct mlx5_fpga_dma_buf *buf)
84 {
85 struct device *dma_device;
86
87 dma_device = &conn->fdev->mdev->pdev->dev;
88 if (buf->sg[1].data)
89 dma_unmap_single(dma_device, buf->sg[1].dma_addr,
90 buf->sg[1].size, buf->dma_dir);
91
92 if (likely(buf->sg[0].data))
93 dma_unmap_single(dma_device, buf->sg[0].dma_addr,
94 buf->sg[0].size, buf->dma_dir);
95 }
96
mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)97 static int mlx5_fpga_conn_post_recv(struct mlx5_fpga_conn *conn,
98 struct mlx5_fpga_dma_buf *buf)
99 {
100 struct mlx5_wqe_data_seg *data;
101 unsigned int ix;
102 int err = 0;
103
104 err = mlx5_fpga_conn_map_buf(conn, buf);
105 if (unlikely(err))
106 goto out;
107
108 if (unlikely(conn->qp.rq.pc - conn->qp.rq.cc >= conn->qp.rq.size)) {
109 mlx5_fpga_conn_unmap_buf(conn, buf);
110 return -EBUSY;
111 }
112
113 ix = conn->qp.rq.pc & (conn->qp.rq.size - 1);
114 data = mlx5_wq_cyc_get_wqe(&conn->qp.wq.rq, ix);
115 data->byte_count = cpu_to_be32(buf->sg[0].size);
116 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
117 data->addr = cpu_to_be64(buf->sg[0].dma_addr);
118
119 conn->qp.rq.pc++;
120 conn->qp.rq.bufs[ix] = buf;
121
122 /* Make sure that descriptors are written before doorbell record. */
123 dma_wmb();
124 *conn->qp.wq.rq.db = cpu_to_be32(conn->qp.rq.pc & 0xffff);
125 out:
126 return err;
127 }
128
mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn * conn,void * wqe)129 static void mlx5_fpga_conn_notify_hw(struct mlx5_fpga_conn *conn, void *wqe)
130 {
131 /* ensure wqe is visible to device before updating doorbell record */
132 dma_wmb();
133 *conn->qp.wq.sq.db = cpu_to_be32(conn->qp.sq.pc);
134 /* Make sure that doorbell record is visible before ringing */
135 wmb();
136 mlx5_write64(wqe, conn->fdev->conn_res.uar->map + MLX5_BF_OFFSET, NULL);
137 }
138
mlx5_fpga_conn_post_send(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)139 static void mlx5_fpga_conn_post_send(struct mlx5_fpga_conn *conn,
140 struct mlx5_fpga_dma_buf *buf)
141 {
142 struct mlx5_wqe_ctrl_seg *ctrl;
143 struct mlx5_wqe_data_seg *data;
144 unsigned int ix, sgi;
145 int size = 1;
146
147 ix = conn->qp.sq.pc & (conn->qp.sq.size - 1);
148
149 ctrl = mlx5_wq_cyc_get_wqe(&conn->qp.wq.sq, ix);
150 data = (void *)(ctrl + 1);
151
152 for (sgi = 0; sgi < ARRAY_SIZE(buf->sg); sgi++) {
153 if (!buf->sg[sgi].data)
154 break;
155 data->byte_count = cpu_to_be32(buf->sg[sgi].size);
156 data->lkey = cpu_to_be32(conn->fdev->conn_res.mkey.key);
157 data->addr = cpu_to_be64(buf->sg[sgi].dma_addr);
158 data++;
159 size++;
160 }
161
162 ctrl->imm = 0;
163 ctrl->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
164 ctrl->opmod_idx_opcode = cpu_to_be32(((conn->qp.sq.pc & 0xffff) << 8) |
165 MLX5_OPCODE_SEND);
166 ctrl->qpn_ds = cpu_to_be32(size | (conn->qp.mqp.qpn << 8));
167
168 conn->qp.sq.pc++;
169 conn->qp.sq.bufs[ix] = buf;
170 mlx5_fpga_conn_notify_hw(conn, ctrl);
171 }
172
mlx5_fpga_conn_send(struct mlx5_fpga_conn * conn,struct mlx5_fpga_dma_buf * buf)173 int mlx5_fpga_conn_send(struct mlx5_fpga_conn *conn,
174 struct mlx5_fpga_dma_buf *buf)
175 {
176 unsigned long flags;
177 int err;
178
179 if (!conn->qp.active)
180 return -ENOTCONN;
181
182 err = mlx5_fpga_conn_map_buf(conn, buf);
183 if (err)
184 return err;
185
186 spin_lock_irqsave(&conn->qp.sq.lock, flags);
187
188 if (conn->qp.sq.pc - conn->qp.sq.cc >= conn->qp.sq.size) {
189 list_add_tail(&buf->list, &conn->qp.sq.backlog);
190 goto out_unlock;
191 }
192
193 mlx5_fpga_conn_post_send(conn, buf);
194
195 out_unlock:
196 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
197 return err;
198 }
199
mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn * conn)200 static int mlx5_fpga_conn_post_recv_buf(struct mlx5_fpga_conn *conn)
201 {
202 struct mlx5_fpga_dma_buf *buf;
203 int err;
204
205 buf = kzalloc(sizeof(*buf) + MLX5_FPGA_RECV_SIZE, 0);
206 if (!buf)
207 return -ENOMEM;
208
209 buf->sg[0].data = (void *)(buf + 1);
210 buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
211 buf->dma_dir = DMA_FROM_DEVICE;
212
213 err = mlx5_fpga_conn_post_recv(conn, buf);
214 if (err)
215 kfree(buf);
216
217 return err;
218 }
219
mlx5_fpga_conn_create_mkey(struct mlx5_core_dev * mdev,u32 pdn,struct mlx5_core_mkey * mkey)220 static int mlx5_fpga_conn_create_mkey(struct mlx5_core_dev *mdev, u32 pdn,
221 struct mlx5_core_mkey *mkey)
222 {
223 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
224 void *mkc;
225 u32 *in;
226 int err;
227
228 in = kvzalloc(inlen, GFP_KERNEL);
229 if (!in)
230 return -ENOMEM;
231
232 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
233 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA);
234 MLX5_SET(mkc, mkc, lw, 1);
235 MLX5_SET(mkc, mkc, lr, 1);
236
237 MLX5_SET(mkc, mkc, pd, pdn);
238 MLX5_SET(mkc, mkc, length64, 1);
239 MLX5_SET(mkc, mkc, qpn, 0xffffff);
240
241 err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
242
243 kvfree(in);
244 return err;
245 }
246
mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn * conn,struct mlx5_cqe64 * cqe,u8 status)247 static void mlx5_fpga_conn_rq_cqe(struct mlx5_fpga_conn *conn,
248 struct mlx5_cqe64 *cqe, u8 status)
249 {
250 struct mlx5_fpga_dma_buf *buf;
251 int ix, err;
252
253 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.rq.size - 1);
254 buf = conn->qp.rq.bufs[ix];
255 conn->qp.rq.bufs[ix] = NULL;
256 if (!status)
257 buf->sg[0].size = be32_to_cpu(cqe->byte_cnt);
258 conn->qp.rq.cc++;
259
260 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
261 mlx5_fpga_warn(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
262 buf, conn->fpga_qpn, status);
263 else
264 mlx5_fpga_dbg(conn->fdev, "RQ buf %p on FPGA QP %u completion status %d\n",
265 buf, conn->fpga_qpn, status);
266
267 mlx5_fpga_conn_unmap_buf(conn, buf);
268
269 if (unlikely(status || !conn->qp.active)) {
270 conn->qp.active = false;
271 kfree(buf);
272 return;
273 }
274
275 mlx5_fpga_dbg(conn->fdev, "Message with %u bytes received successfully\n",
276 buf->sg[0].size);
277 conn->recv_cb(conn->cb_arg, buf);
278
279 buf->sg[0].size = MLX5_FPGA_RECV_SIZE;
280 err = mlx5_fpga_conn_post_recv(conn, buf);
281 if (unlikely(err)) {
282 mlx5_fpga_warn(conn->fdev,
283 "Failed to re-post recv buf: %d\n", err);
284 kfree(buf);
285 }
286 }
287
mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn * conn,struct mlx5_cqe64 * cqe,u8 status)288 static void mlx5_fpga_conn_sq_cqe(struct mlx5_fpga_conn *conn,
289 struct mlx5_cqe64 *cqe, u8 status)
290 {
291 struct mlx5_fpga_dma_buf *buf, *nextbuf;
292 unsigned long flags;
293 int ix;
294
295 spin_lock_irqsave(&conn->qp.sq.lock, flags);
296
297 ix = be16_to_cpu(cqe->wqe_counter) & (conn->qp.sq.size - 1);
298 buf = conn->qp.sq.bufs[ix];
299 conn->qp.sq.bufs[ix] = NULL;
300 conn->qp.sq.cc++;
301
302 /* Handle backlog still under the spinlock to ensure message post order */
303 if (unlikely(!list_empty(&conn->qp.sq.backlog))) {
304 if (likely(conn->qp.active)) {
305 nextbuf = list_first_entry(&conn->qp.sq.backlog,
306 struct mlx5_fpga_dma_buf, list);
307 list_del(&nextbuf->list);
308 mlx5_fpga_conn_post_send(conn, nextbuf);
309 }
310 }
311
312 spin_unlock_irqrestore(&conn->qp.sq.lock, flags);
313
314 if (unlikely(status && (status != MLX5_CQE_SYNDROME_WR_FLUSH_ERR)))
315 mlx5_fpga_warn(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
316 buf, conn->fpga_qpn, status);
317 else
318 mlx5_fpga_dbg(conn->fdev, "SQ buf %p on FPGA QP %u completion status %d\n",
319 buf, conn->fpga_qpn, status);
320
321 mlx5_fpga_conn_unmap_buf(conn, buf);
322
323 if (likely(buf->complete))
324 buf->complete(conn, conn->fdev, buf, status);
325
326 if (unlikely(status))
327 conn->qp.active = false;
328 }
329
mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn * conn,struct mlx5_cqe64 * cqe)330 static void mlx5_fpga_conn_handle_cqe(struct mlx5_fpga_conn *conn,
331 struct mlx5_cqe64 *cqe)
332 {
333 u8 opcode, status = 0;
334
335 opcode = cqe->op_own >> 4;
336
337 switch (opcode) {
338 case MLX5_CQE_REQ_ERR:
339 status = ((struct mlx5_err_cqe *)cqe)->syndrome;
340 /* Fall through */
341 case MLX5_CQE_REQ:
342 mlx5_fpga_conn_sq_cqe(conn, cqe, status);
343 break;
344
345 case MLX5_CQE_RESP_ERR:
346 status = ((struct mlx5_err_cqe *)cqe)->syndrome;
347 /* Fall through */
348 case MLX5_CQE_RESP_SEND:
349 mlx5_fpga_conn_rq_cqe(conn, cqe, status);
350 break;
351 default:
352 mlx5_fpga_warn(conn->fdev, "Unexpected cqe opcode %u\n",
353 opcode);
354 }
355 }
356
mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn * conn)357 static void mlx5_fpga_conn_arm_cq(struct mlx5_fpga_conn *conn)
358 {
359 mlx5_cq_arm(&conn->cq.mcq, MLX5_CQ_DB_REQ_NOT,
360 conn->fdev->conn_res.uar->map, conn->cq.wq.cc);
361 }
362
mlx5_fpga_conn_cq_event(struct mlx5_core_cq * mcq,enum mlx5_event event)363 static void mlx5_fpga_conn_cq_event(struct mlx5_core_cq *mcq,
364 enum mlx5_event event)
365 {
366 struct mlx5_fpga_conn *conn;
367
368 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
369 mlx5_fpga_warn(conn->fdev, "CQ event %u on CQ #%u\n", event, mcq->cqn);
370 }
371
mlx5_fpga_conn_event(struct mlx5_core_qp * mqp,int event)372 static void mlx5_fpga_conn_event(struct mlx5_core_qp *mqp, int event)
373 {
374 struct mlx5_fpga_conn *conn;
375
376 conn = container_of(mqp, struct mlx5_fpga_conn, qp.mqp);
377 mlx5_fpga_warn(conn->fdev, "QP event %u on QP #%u\n", event, mqp->qpn);
378 }
379
mlx5_fpga_conn_cqes(struct mlx5_fpga_conn * conn,unsigned int budget)380 static inline void mlx5_fpga_conn_cqes(struct mlx5_fpga_conn *conn,
381 unsigned int budget)
382 {
383 struct mlx5_cqe64 *cqe;
384
385 while (budget) {
386 cqe = mlx5_cqwq_get_cqe(&conn->cq.wq);
387 if (!cqe)
388 break;
389
390 budget--;
391 mlx5_cqwq_pop(&conn->cq.wq);
392 mlx5_fpga_conn_handle_cqe(conn, cqe);
393 mlx5_cqwq_update_db_record(&conn->cq.wq);
394 }
395 if (!budget) {
396 tasklet_schedule(&conn->cq.tasklet);
397 return;
398 }
399
400 mlx5_fpga_dbg(conn->fdev, "Re-arming CQ with cc# %u\n", conn->cq.wq.cc);
401 /* ensure cq space is freed before enabling more cqes */
402 wmb();
403 mlx5_fpga_conn_arm_cq(conn);
404 }
405
mlx5_fpga_conn_cq_tasklet(unsigned long data)406 static void mlx5_fpga_conn_cq_tasklet(unsigned long data)
407 {
408 struct mlx5_fpga_conn *conn = (void *)data;
409
410 if (unlikely(!conn->qp.active))
411 return;
412 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
413 }
414
mlx5_fpga_conn_cq_complete(struct mlx5_core_cq * mcq)415 static void mlx5_fpga_conn_cq_complete(struct mlx5_core_cq *mcq)
416 {
417 struct mlx5_fpga_conn *conn;
418
419 conn = container_of(mcq, struct mlx5_fpga_conn, cq.mcq);
420 if (unlikely(!conn->qp.active))
421 return;
422 mlx5_fpga_conn_cqes(conn, MLX5_FPGA_CQ_BUDGET);
423 }
424
mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn * conn,int cq_size)425 static int mlx5_fpga_conn_create_cq(struct mlx5_fpga_conn *conn, int cq_size)
426 {
427 struct mlx5_fpga_device *fdev = conn->fdev;
428 struct mlx5_core_dev *mdev = fdev->mdev;
429 u32 temp_cqc[MLX5_ST_SZ_DW(cqc)] = {0};
430 struct mlx5_wq_param wqp;
431 struct mlx5_cqe64 *cqe;
432 int inlen, err, eqn;
433 unsigned int irqn;
434 void *cqc, *in;
435 __be64 *pas;
436 u32 i;
437
438 cq_size = roundup_pow_of_two(cq_size);
439 MLX5_SET(cqc, temp_cqc, log_cq_size, ilog2(cq_size));
440
441 wqp.buf_numa_node = mdev->priv.numa_node;
442 wqp.db_numa_node = mdev->priv.numa_node;
443
444 err = mlx5_cqwq_create(mdev, &wqp, temp_cqc, &conn->cq.wq,
445 &conn->cq.wq_ctrl);
446 if (err)
447 return err;
448
449 for (i = 0; i < mlx5_cqwq_get_size(&conn->cq.wq); i++) {
450 cqe = mlx5_cqwq_get_wqe(&conn->cq.wq, i);
451 cqe->op_own = MLX5_CQE_INVALID << 4 | MLX5_CQE_OWNER_MASK;
452 }
453
454 inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
455 sizeof(u64) * conn->cq.wq_ctrl.frag_buf.npages;
456 in = kvzalloc(inlen, GFP_KERNEL);
457 if (!in) {
458 err = -ENOMEM;
459 goto err_cqwq;
460 }
461
462 err = mlx5_vector2eqn(mdev, smp_processor_id(), &eqn, &irqn);
463 if (err)
464 goto err_cqwq;
465
466 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
467 MLX5_SET(cqc, cqc, log_cq_size, ilog2(cq_size));
468 MLX5_SET(cqc, cqc, c_eqn, eqn);
469 MLX5_SET(cqc, cqc, uar_page, fdev->conn_res.uar->index);
470 MLX5_SET(cqc, cqc, log_page_size, conn->cq.wq_ctrl.frag_buf.page_shift -
471 MLX5_ADAPTER_PAGE_SHIFT);
472 MLX5_SET64(cqc, cqc, dbr_addr, conn->cq.wq_ctrl.db.dma);
473
474 pas = (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas);
475 mlx5_fill_page_frag_array(&conn->cq.wq_ctrl.frag_buf, pas);
476
477 err = mlx5_core_create_cq(mdev, &conn->cq.mcq, in, inlen);
478 kvfree(in);
479
480 if (err)
481 goto err_cqwq;
482
483 conn->cq.mcq.cqe_sz = 64;
484 conn->cq.mcq.set_ci_db = conn->cq.wq_ctrl.db.db;
485 conn->cq.mcq.arm_db = conn->cq.wq_ctrl.db.db + 1;
486 *conn->cq.mcq.set_ci_db = 0;
487 *conn->cq.mcq.arm_db = 0;
488 conn->cq.mcq.vector = 0;
489 conn->cq.mcq.comp = mlx5_fpga_conn_cq_complete;
490 conn->cq.mcq.event = mlx5_fpga_conn_cq_event;
491 conn->cq.mcq.irqn = irqn;
492 conn->cq.mcq.uar = fdev->conn_res.uar;
493 tasklet_init(&conn->cq.tasklet, mlx5_fpga_conn_cq_tasklet,
494 (unsigned long)conn);
495
496 mlx5_fpga_dbg(fdev, "Created CQ #0x%x\n", conn->cq.mcq.cqn);
497
498 goto out;
499
500 err_cqwq:
501 mlx5_cqwq_destroy(&conn->cq.wq_ctrl);
502 out:
503 return err;
504 }
505
mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn * conn)506 static void mlx5_fpga_conn_destroy_cq(struct mlx5_fpga_conn *conn)
507 {
508 tasklet_disable(&conn->cq.tasklet);
509 tasklet_kill(&conn->cq.tasklet);
510 mlx5_core_destroy_cq(conn->fdev->mdev, &conn->cq.mcq);
511 mlx5_cqwq_destroy(&conn->cq.wq_ctrl);
512 }
513
mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn * conn,void * qpc)514 static int mlx5_fpga_conn_create_wq(struct mlx5_fpga_conn *conn, void *qpc)
515 {
516 struct mlx5_fpga_device *fdev = conn->fdev;
517 struct mlx5_core_dev *mdev = fdev->mdev;
518 struct mlx5_wq_param wqp;
519
520 wqp.buf_numa_node = mdev->priv.numa_node;
521 wqp.db_numa_node = mdev->priv.numa_node;
522
523 return mlx5_wq_qp_create(mdev, &wqp, qpc, &conn->qp.wq,
524 &conn->qp.wq_ctrl);
525 }
526
mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn * conn,unsigned int tx_size,unsigned int rx_size)527 static int mlx5_fpga_conn_create_qp(struct mlx5_fpga_conn *conn,
528 unsigned int tx_size, unsigned int rx_size)
529 {
530 struct mlx5_fpga_device *fdev = conn->fdev;
531 struct mlx5_core_dev *mdev = fdev->mdev;
532 u32 temp_qpc[MLX5_ST_SZ_DW(qpc)] = {0};
533 void *in = NULL, *qpc;
534 int err, inlen;
535
536 conn->qp.rq.pc = 0;
537 conn->qp.rq.cc = 0;
538 conn->qp.rq.size = roundup_pow_of_two(rx_size);
539 conn->qp.sq.pc = 0;
540 conn->qp.sq.cc = 0;
541 conn->qp.sq.size = roundup_pow_of_two(tx_size);
542
543 MLX5_SET(qpc, temp_qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
544 MLX5_SET(qpc, temp_qpc, log_rq_size, ilog2(conn->qp.rq.size));
545 MLX5_SET(qpc, temp_qpc, log_sq_size, ilog2(conn->qp.sq.size));
546 err = mlx5_fpga_conn_create_wq(conn, temp_qpc);
547 if (err)
548 goto out;
549
550 conn->qp.rq.bufs = kvzalloc(sizeof(conn->qp.rq.bufs[0]) *
551 conn->qp.rq.size, GFP_KERNEL);
552 if (!conn->qp.rq.bufs) {
553 err = -ENOMEM;
554 goto err_wq;
555 }
556
557 conn->qp.sq.bufs = kvzalloc(sizeof(conn->qp.sq.bufs[0]) *
558 conn->qp.sq.size, GFP_KERNEL);
559 if (!conn->qp.sq.bufs) {
560 err = -ENOMEM;
561 goto err_rq_bufs;
562 }
563
564 inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
565 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) *
566 conn->qp.wq_ctrl.buf.npages;
567 in = kvzalloc(inlen, GFP_KERNEL);
568 if (!in) {
569 err = -ENOMEM;
570 goto err_sq_bufs;
571 }
572
573 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
574 MLX5_SET(qpc, qpc, uar_page, fdev->conn_res.uar->index);
575 MLX5_SET(qpc, qpc, log_page_size,
576 conn->qp.wq_ctrl.buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
577 MLX5_SET(qpc, qpc, fre, 1);
578 MLX5_SET(qpc, qpc, rlky, 1);
579 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
580 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
581 MLX5_SET(qpc, qpc, pd, fdev->conn_res.pdn);
582 MLX5_SET(qpc, qpc, log_rq_stride, ilog2(MLX5_SEND_WQE_DS) - 4);
583 MLX5_SET(qpc, qpc, log_rq_size, ilog2(conn->qp.rq.size));
584 MLX5_SET(qpc, qpc, rq_type, MLX5_NON_ZERO_RQ);
585 MLX5_SET(qpc, qpc, log_sq_size, ilog2(conn->qp.sq.size));
586 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
587 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
588 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
589 if (MLX5_CAP_GEN(mdev, cqe_version) == 1)
590 MLX5_SET(qpc, qpc, user_index, 0xFFFFFF);
591
592 mlx5_fill_page_array(&conn->qp.wq_ctrl.buf,
593 (__be64 *)MLX5_ADDR_OF(create_qp_in, in, pas));
594
595 err = mlx5_core_create_qp(mdev, &conn->qp.mqp, in, inlen);
596 if (err)
597 goto err_sq_bufs;
598
599 conn->qp.mqp.event = mlx5_fpga_conn_event;
600 mlx5_fpga_dbg(fdev, "Created QP #0x%x\n", conn->qp.mqp.qpn);
601
602 goto out;
603
604 err_sq_bufs:
605 kvfree(conn->qp.sq.bufs);
606 err_rq_bufs:
607 kvfree(conn->qp.rq.bufs);
608 err_wq:
609 mlx5_wq_destroy(&conn->qp.wq_ctrl);
610 out:
611 kvfree(in);
612 return err;
613 }
614
mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn * conn)615 static void mlx5_fpga_conn_free_recv_bufs(struct mlx5_fpga_conn *conn)
616 {
617 int ix;
618
619 for (ix = 0; ix < conn->qp.rq.size; ix++) {
620 if (!conn->qp.rq.bufs[ix])
621 continue;
622 mlx5_fpga_conn_unmap_buf(conn, conn->qp.rq.bufs[ix]);
623 kfree(conn->qp.rq.bufs[ix]);
624 conn->qp.rq.bufs[ix] = NULL;
625 }
626 }
627
mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn * conn)628 static void mlx5_fpga_conn_flush_send_bufs(struct mlx5_fpga_conn *conn)
629 {
630 struct mlx5_fpga_dma_buf *buf, *temp;
631 int ix;
632
633 for (ix = 0; ix < conn->qp.sq.size; ix++) {
634 buf = conn->qp.sq.bufs[ix];
635 if (!buf)
636 continue;
637 conn->qp.sq.bufs[ix] = NULL;
638 mlx5_fpga_conn_unmap_buf(conn, buf);
639 if (!buf->complete)
640 continue;
641 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
642 }
643 list_for_each_entry_safe(buf, temp, &conn->qp.sq.backlog, list) {
644 mlx5_fpga_conn_unmap_buf(conn, buf);
645 if (!buf->complete)
646 continue;
647 buf->complete(conn, conn->fdev, buf, MLX5_CQE_SYNDROME_WR_FLUSH_ERR);
648 }
649 }
650
mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn * conn)651 static void mlx5_fpga_conn_destroy_qp(struct mlx5_fpga_conn *conn)
652 {
653 mlx5_core_destroy_qp(conn->fdev->mdev, &conn->qp.mqp);
654 mlx5_fpga_conn_free_recv_bufs(conn);
655 mlx5_fpga_conn_flush_send_bufs(conn);
656 kvfree(conn->qp.sq.bufs);
657 kvfree(conn->qp.rq.bufs);
658 mlx5_wq_destroy(&conn->qp.wq_ctrl);
659 }
660
mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn * conn)661 static inline int mlx5_fpga_conn_reset_qp(struct mlx5_fpga_conn *conn)
662 {
663 struct mlx5_core_dev *mdev = conn->fdev->mdev;
664
665 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to RST\n", conn->qp.mqp.qpn);
666
667 return mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2RST_QP, 0, NULL,
668 &conn->qp.mqp);
669 }
670
mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn * conn)671 static inline int mlx5_fpga_conn_init_qp(struct mlx5_fpga_conn *conn)
672 {
673 struct mlx5_fpga_device *fdev = conn->fdev;
674 struct mlx5_core_dev *mdev = fdev->mdev;
675 u32 *qpc = NULL;
676 int err;
677
678 mlx5_fpga_dbg(conn->fdev, "Modifying QP %u to INIT\n", conn->qp.mqp.qpn);
679
680 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
681 if (!qpc) {
682 err = -ENOMEM;
683 goto out;
684 }
685
686 MLX5_SET(qpc, qpc, st, MLX5_QP_ST_RC);
687 MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
688 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
689 MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM);
690 MLX5_SET(qpc, qpc, pd, conn->fdev->conn_res.pdn);
691 MLX5_SET(qpc, qpc, cqn_snd, conn->cq.mcq.cqn);
692 MLX5_SET(qpc, qpc, cqn_rcv, conn->cq.mcq.cqn);
693 MLX5_SET64(qpc, qpc, dbr_addr, conn->qp.wq_ctrl.db.dma);
694
695 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RST2INIT_QP, 0, qpc,
696 &conn->qp.mqp);
697 if (err) {
698 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err);
699 goto out;
700 }
701
702 out:
703 kfree(qpc);
704 return err;
705 }
706
mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn * conn)707 static inline int mlx5_fpga_conn_rtr_qp(struct mlx5_fpga_conn *conn)
708 {
709 struct mlx5_fpga_device *fdev = conn->fdev;
710 struct mlx5_core_dev *mdev = fdev->mdev;
711 u32 *qpc = NULL;
712 int err;
713
714 mlx5_fpga_dbg(conn->fdev, "QP RTR\n");
715
716 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
717 if (!qpc) {
718 err = -ENOMEM;
719 goto out;
720 }
721
722 MLX5_SET(qpc, qpc, mtu, MLX5_QPC_MTU_1K_BYTES);
723 MLX5_SET(qpc, qpc, log_msg_max, (u8)MLX5_CAP_GEN(mdev, log_max_msg));
724 MLX5_SET(qpc, qpc, remote_qpn, conn->fpga_qpn);
725 MLX5_SET(qpc, qpc, next_rcv_psn,
726 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_send_psn));
727 MLX5_SET(qpc, qpc, primary_address_path.pkey_index, MLX5_FPGA_PKEY_INDEX);
728 MLX5_SET(qpc, qpc, primary_address_path.port, MLX5_FPGA_PORT_NUM);
729 ether_addr_copy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rmac_47_32),
730 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_mac_47_32));
731 MLX5_SET(qpc, qpc, primary_address_path.udp_sport,
732 MLX5_CAP_ROCE(mdev, r_roce_min_src_udp_port));
733 MLX5_SET(qpc, qpc, primary_address_path.src_addr_index,
734 conn->qp.sgid_index);
735 MLX5_SET(qpc, qpc, primary_address_path.hop_limit, 0);
736 memcpy(MLX5_ADDR_OF(qpc, qpc, primary_address_path.rgid_rip),
737 MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, fpga_ip),
738 MLX5_FLD_SZ_BYTES(qpc, primary_address_path.rgid_rip));
739
740 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_INIT2RTR_QP, 0, qpc,
741 &conn->qp.mqp);
742 if (err) {
743 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err);
744 goto out;
745 }
746
747 out:
748 kfree(qpc);
749 return err;
750 }
751
mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn * conn)752 static inline int mlx5_fpga_conn_rts_qp(struct mlx5_fpga_conn *conn)
753 {
754 struct mlx5_fpga_device *fdev = conn->fdev;
755 struct mlx5_core_dev *mdev = fdev->mdev;
756 u32 *qpc = NULL;
757 u32 opt_mask;
758 int err;
759
760 mlx5_fpga_dbg(conn->fdev, "QP RTS\n");
761
762 qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
763 if (!qpc) {
764 err = -ENOMEM;
765 goto out;
766 }
767
768 MLX5_SET(qpc, qpc, log_ack_req_freq, 8);
769 MLX5_SET(qpc, qpc, min_rnr_nak, 0x12);
770 MLX5_SET(qpc, qpc, primary_address_path.ack_timeout, 0x12); /* ~1.07s */
771 MLX5_SET(qpc, qpc, next_send_psn,
772 MLX5_GET(fpga_qpc, conn->fpga_qpc, next_rcv_psn));
773 MLX5_SET(qpc, qpc, retry_count, 7);
774 MLX5_SET(qpc, qpc, rnr_retry, 7); /* Infinite retry if RNR NACK */
775
776 opt_mask = MLX5_QP_OPTPAR_RNR_TIMEOUT;
777 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_RTR2RTS_QP, opt_mask, qpc,
778 &conn->qp.mqp);
779 if (err) {
780 mlx5_fpga_warn(fdev, "qp_modify RST2INIT failed: %d\n", err);
781 goto out;
782 }
783
784 out:
785 kfree(qpc);
786 return err;
787 }
788
mlx5_fpga_conn_connect(struct mlx5_fpga_conn * conn)789 static int mlx5_fpga_conn_connect(struct mlx5_fpga_conn *conn)
790 {
791 struct mlx5_fpga_device *fdev = conn->fdev;
792 int err;
793
794 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_ACTIVE);
795 err = mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
796 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc);
797 if (err) {
798 mlx5_fpga_err(fdev, "Failed to activate FPGA RC QP: %d\n", err);
799 goto out;
800 }
801
802 err = mlx5_fpga_conn_reset_qp(conn);
803 if (err) {
804 mlx5_fpga_err(fdev, "Failed to change QP state to reset\n");
805 goto err_fpga_qp;
806 }
807
808 err = mlx5_fpga_conn_init_qp(conn);
809 if (err) {
810 mlx5_fpga_err(fdev, "Failed to modify QP from RESET to INIT\n");
811 goto err_fpga_qp;
812 }
813 conn->qp.active = true;
814
815 while (!mlx5_fpga_conn_post_recv_buf(conn))
816 ;
817
818 err = mlx5_fpga_conn_rtr_qp(conn);
819 if (err) {
820 mlx5_fpga_err(fdev, "Failed to change QP state from INIT to RTR\n");
821 goto err_recv_bufs;
822 }
823
824 err = mlx5_fpga_conn_rts_qp(conn);
825 if (err) {
826 mlx5_fpga_err(fdev, "Failed to change QP state from RTR to RTS\n");
827 goto err_recv_bufs;
828 }
829 goto out;
830
831 err_recv_bufs:
832 mlx5_fpga_conn_free_recv_bufs(conn);
833 err_fpga_qp:
834 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
835 if (mlx5_fpga_modify_qp(conn->fdev->mdev, conn->fpga_qpn,
836 MLX5_FPGA_QPC_STATE, &conn->fpga_qpc))
837 mlx5_fpga_err(fdev, "Failed to revert FPGA QP to INIT\n");
838 out:
839 return err;
840 }
841
mlx5_fpga_conn_create(struct mlx5_fpga_device * fdev,struct mlx5_fpga_conn_attr * attr,enum mlx5_ifc_fpga_qp_type qp_type)842 struct mlx5_fpga_conn *mlx5_fpga_conn_create(struct mlx5_fpga_device *fdev,
843 struct mlx5_fpga_conn_attr *attr,
844 enum mlx5_ifc_fpga_qp_type qp_type)
845 {
846 struct mlx5_fpga_conn *ret, *conn;
847 u8 *remote_mac, *remote_ip;
848 int err;
849
850 if (!attr->recv_cb)
851 return ERR_PTR(-EINVAL);
852
853 conn = kzalloc(sizeof(*conn), GFP_KERNEL);
854 if (!conn)
855 return ERR_PTR(-ENOMEM);
856
857 conn->fdev = fdev;
858 INIT_LIST_HEAD(&conn->qp.sq.backlog);
859
860 spin_lock_init(&conn->qp.sq.lock);
861
862 conn->recv_cb = attr->recv_cb;
863 conn->cb_arg = attr->cb_arg;
864
865 remote_mac = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_mac_47_32);
866 err = mlx5_query_nic_vport_mac_address(fdev->mdev, 0, remote_mac);
867 if (err) {
868 mlx5_fpga_err(fdev, "Failed to query local MAC: %d\n", err);
869 ret = ERR_PTR(err);
870 goto err;
871 }
872
873 /* Build Modified EUI-64 IPv6 address from the MAC address */
874 remote_ip = MLX5_ADDR_OF(fpga_qpc, conn->fpga_qpc, remote_ip);
875 remote_ip[0] = 0xfe;
876 remote_ip[1] = 0x80;
877 addrconf_addr_eui48(&remote_ip[8], remote_mac);
878
879 err = mlx5_core_reserved_gid_alloc(fdev->mdev, &conn->qp.sgid_index);
880 if (err) {
881 mlx5_fpga_err(fdev, "Failed to allocate SGID: %d\n", err);
882 ret = ERR_PTR(err);
883 goto err;
884 }
885
886 err = mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index,
887 MLX5_ROCE_VERSION_2,
888 MLX5_ROCE_L3_TYPE_IPV6,
889 remote_ip, remote_mac, true, 0);
890 if (err) {
891 mlx5_fpga_err(fdev, "Failed to set SGID: %d\n", err);
892 ret = ERR_PTR(err);
893 goto err_rsvd_gid;
894 }
895 mlx5_fpga_dbg(fdev, "Reserved SGID index %u\n", conn->qp.sgid_index);
896
897 /* Allow for one cqe per rx/tx wqe, plus one cqe for the next wqe,
898 * created during processing of the cqe
899 */
900 err = mlx5_fpga_conn_create_cq(conn,
901 (attr->tx_size + attr->rx_size) * 2);
902 if (err) {
903 mlx5_fpga_err(fdev, "Failed to create CQ: %d\n", err);
904 ret = ERR_PTR(err);
905 goto err_gid;
906 }
907
908 mlx5_fpga_conn_arm_cq(conn);
909
910 err = mlx5_fpga_conn_create_qp(conn, attr->tx_size, attr->rx_size);
911 if (err) {
912 mlx5_fpga_err(fdev, "Failed to create QP: %d\n", err);
913 ret = ERR_PTR(err);
914 goto err_cq;
915 }
916
917 MLX5_SET(fpga_qpc, conn->fpga_qpc, state, MLX5_FPGA_QPC_STATE_INIT);
918 MLX5_SET(fpga_qpc, conn->fpga_qpc, qp_type, qp_type);
919 MLX5_SET(fpga_qpc, conn->fpga_qpc, st, MLX5_FPGA_QPC_ST_RC);
920 MLX5_SET(fpga_qpc, conn->fpga_qpc, ether_type, ETH_P_8021Q);
921 MLX5_SET(fpga_qpc, conn->fpga_qpc, vid, 0);
922 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_rcv_psn, 1);
923 MLX5_SET(fpga_qpc, conn->fpga_qpc, next_send_psn, 0);
924 MLX5_SET(fpga_qpc, conn->fpga_qpc, pkey, MLX5_FPGA_PKEY);
925 MLX5_SET(fpga_qpc, conn->fpga_qpc, remote_qpn, conn->qp.mqp.qpn);
926 MLX5_SET(fpga_qpc, conn->fpga_qpc, rnr_retry, 7);
927 MLX5_SET(fpga_qpc, conn->fpga_qpc, retry_count, 7);
928
929 err = mlx5_fpga_create_qp(fdev->mdev, &conn->fpga_qpc,
930 &conn->fpga_qpn);
931 if (err) {
932 mlx5_fpga_err(fdev, "Failed to create FPGA RC QP: %d\n", err);
933 ret = ERR_PTR(err);
934 goto err_qp;
935 }
936
937 err = mlx5_fpga_conn_connect(conn);
938 if (err) {
939 ret = ERR_PTR(err);
940 goto err_conn;
941 }
942
943 mlx5_fpga_dbg(fdev, "FPGA QPN is %u\n", conn->fpga_qpn);
944 ret = conn;
945 goto out;
946
947 err_conn:
948 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
949 err_qp:
950 mlx5_fpga_conn_destroy_qp(conn);
951 err_cq:
952 mlx5_fpga_conn_destroy_cq(conn);
953 err_gid:
954 mlx5_core_roce_gid_set(fdev->mdev, conn->qp.sgid_index, 0, 0, NULL,
955 NULL, false, 0);
956 err_rsvd_gid:
957 mlx5_core_reserved_gid_free(fdev->mdev, conn->qp.sgid_index);
958 err:
959 kfree(conn);
960 out:
961 return ret;
962 }
963
mlx5_fpga_conn_destroy(struct mlx5_fpga_conn * conn)964 void mlx5_fpga_conn_destroy(struct mlx5_fpga_conn *conn)
965 {
966 struct mlx5_fpga_device *fdev = conn->fdev;
967 struct mlx5_core_dev *mdev = fdev->mdev;
968 int err = 0;
969
970 conn->qp.active = false;
971 tasklet_disable(&conn->cq.tasklet);
972 synchronize_irq(conn->cq.mcq.irqn);
973
974 mlx5_fpga_destroy_qp(conn->fdev->mdev, conn->fpga_qpn);
975 err = mlx5_core_qp_modify(mdev, MLX5_CMD_OP_2ERR_QP, 0, NULL,
976 &conn->qp.mqp);
977 if (err)
978 mlx5_fpga_warn(fdev, "qp_modify 2ERR failed: %d\n", err);
979 mlx5_fpga_conn_destroy_qp(conn);
980 mlx5_fpga_conn_destroy_cq(conn);
981
982 mlx5_core_roce_gid_set(conn->fdev->mdev, conn->qp.sgid_index, 0, 0,
983 NULL, NULL, false, 0);
984 mlx5_core_reserved_gid_free(conn->fdev->mdev, conn->qp.sgid_index);
985 kfree(conn);
986 }
987
mlx5_fpga_conn_device_init(struct mlx5_fpga_device * fdev)988 int mlx5_fpga_conn_device_init(struct mlx5_fpga_device *fdev)
989 {
990 int err;
991
992 err = mlx5_nic_vport_enable_roce(fdev->mdev);
993 if (err) {
994 mlx5_fpga_err(fdev, "Failed to enable RoCE: %d\n", err);
995 goto out;
996 }
997
998 fdev->conn_res.uar = mlx5_get_uars_page(fdev->mdev);
999 if (IS_ERR(fdev->conn_res.uar)) {
1000 err = PTR_ERR(fdev->conn_res.uar);
1001 mlx5_fpga_err(fdev, "get_uars_page failed, %d\n", err);
1002 goto err_roce;
1003 }
1004 mlx5_fpga_dbg(fdev, "Allocated UAR index %u\n",
1005 fdev->conn_res.uar->index);
1006
1007 err = mlx5_core_alloc_pd(fdev->mdev, &fdev->conn_res.pdn);
1008 if (err) {
1009 mlx5_fpga_err(fdev, "alloc pd failed, %d\n", err);
1010 goto err_uar;
1011 }
1012 mlx5_fpga_dbg(fdev, "Allocated PD %u\n", fdev->conn_res.pdn);
1013
1014 err = mlx5_fpga_conn_create_mkey(fdev->mdev, fdev->conn_res.pdn,
1015 &fdev->conn_res.mkey);
1016 if (err) {
1017 mlx5_fpga_err(fdev, "create mkey failed, %d\n", err);
1018 goto err_dealloc_pd;
1019 }
1020 mlx5_fpga_dbg(fdev, "Created mkey 0x%x\n", fdev->conn_res.mkey.key);
1021
1022 return 0;
1023
1024 err_dealloc_pd:
1025 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
1026 err_uar:
1027 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
1028 err_roce:
1029 mlx5_nic_vport_disable_roce(fdev->mdev);
1030 out:
1031 return err;
1032 }
1033
mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device * fdev)1034 void mlx5_fpga_conn_device_cleanup(struct mlx5_fpga_device *fdev)
1035 {
1036 mlx5_core_destroy_mkey(fdev->mdev, &fdev->conn_res.mkey);
1037 mlx5_core_dealloc_pd(fdev->mdev, fdev->conn_res.pdn);
1038 mlx5_put_uars_page(fdev->mdev, fdev->conn_res.uar);
1039 mlx5_nic_vport_disable_roce(fdev->mdev);
1040 }
1041