1f02d0d6eSLeon Romanovsky // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2f02d0d6eSLeon Romanovsky /*
3f02d0d6eSLeon Romanovsky * Copyright (c) 2013-2018, Mellanox Technologies inc. All rights reserved.
4f02d0d6eSLeon Romanovsky */
5f02d0d6eSLeon Romanovsky
6f02d0d6eSLeon Romanovsky #include <linux/kernel.h>
7f02d0d6eSLeon Romanovsky #include <linux/mlx5/driver.h>
8b4990804SLeon Romanovsky #include "mlx5_ib.h"
9f02d0d6eSLeon Romanovsky #include "srq.h"
10333fbaa0SLeon Romanovsky #include "qp.h"
11f02d0d6eSLeon Romanovsky
get_pas_size(struct mlx5_srq_attr * in)12f02d0d6eSLeon Romanovsky static int get_pas_size(struct mlx5_srq_attr *in)
13f02d0d6eSLeon Romanovsky {
14f02d0d6eSLeon Romanovsky u32 log_page_size = in->log_page_size + 12;
15f02d0d6eSLeon Romanovsky u32 log_srq_size = in->log_size;
16f02d0d6eSLeon Romanovsky u32 log_rq_stride = in->wqe_shift;
17f02d0d6eSLeon Romanovsky u32 page_offset = in->page_offset;
18f02d0d6eSLeon Romanovsky u32 po_quanta = 1 << (log_page_size - 6);
19f02d0d6eSLeon Romanovsky u32 rq_sz = 1 << (log_srq_size + 4 + log_rq_stride);
20f02d0d6eSLeon Romanovsky u32 page_size = 1 << log_page_size;
21f02d0d6eSLeon Romanovsky u32 rq_sz_po = rq_sz + (page_offset * po_quanta);
22f02d0d6eSLeon Romanovsky u32 rq_num_pas = DIV_ROUND_UP(rq_sz_po, page_size);
23f02d0d6eSLeon Romanovsky
24f02d0d6eSLeon Romanovsky return rq_num_pas * sizeof(u64);
25f02d0d6eSLeon Romanovsky }
26f02d0d6eSLeon Romanovsky
set_wq(void * wq,struct mlx5_srq_attr * in)27f02d0d6eSLeon Romanovsky static void set_wq(void *wq, struct mlx5_srq_attr *in)
28f02d0d6eSLeon Romanovsky {
29f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, wq_signature, !!(in->flags
30f02d0d6eSLeon Romanovsky & MLX5_SRQ_FLAG_WQ_SIG));
31f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, log_wq_pg_sz, in->log_page_size);
32f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, log_wq_stride, in->wqe_shift + 4);
33f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, log_wq_sz, in->log_size);
34f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, page_offset, in->page_offset);
35f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, lwm, in->lwm);
36f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, pd, in->pd);
37f02d0d6eSLeon Romanovsky MLX5_SET64(wq, wq, dbr_addr, in->db_record);
38f02d0d6eSLeon Romanovsky }
39f02d0d6eSLeon Romanovsky
set_srqc(void * srqc,struct mlx5_srq_attr * in)40f02d0d6eSLeon Romanovsky static void set_srqc(void *srqc, struct mlx5_srq_attr *in)
41f02d0d6eSLeon Romanovsky {
42f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, wq_signature, !!(in->flags
43f02d0d6eSLeon Romanovsky & MLX5_SRQ_FLAG_WQ_SIG));
44f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, log_page_size, in->log_page_size);
45f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, log_rq_stride, in->wqe_shift);
46f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, log_srq_size, in->log_size);
47f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, page_offset, in->page_offset);
48f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, lwm, in->lwm);
49f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, pd, in->pd);
50f02d0d6eSLeon Romanovsky MLX5_SET64(srqc, srqc, dbr_addr, in->db_record);
51f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, xrcd, in->xrcd);
52f02d0d6eSLeon Romanovsky MLX5_SET(srqc, srqc, cqn, in->cqn);
53f02d0d6eSLeon Romanovsky }
54f02d0d6eSLeon Romanovsky
get_wq(void * wq,struct mlx5_srq_attr * in)55f02d0d6eSLeon Romanovsky static void get_wq(void *wq, struct mlx5_srq_attr *in)
56f02d0d6eSLeon Romanovsky {
57f02d0d6eSLeon Romanovsky if (MLX5_GET(wq, wq, wq_signature))
58f02d0d6eSLeon Romanovsky in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
59f02d0d6eSLeon Romanovsky in->log_page_size = MLX5_GET(wq, wq, log_wq_pg_sz);
60f02d0d6eSLeon Romanovsky in->wqe_shift = MLX5_GET(wq, wq, log_wq_stride) - 4;
61f02d0d6eSLeon Romanovsky in->log_size = MLX5_GET(wq, wq, log_wq_sz);
62f02d0d6eSLeon Romanovsky in->page_offset = MLX5_GET(wq, wq, page_offset);
63f02d0d6eSLeon Romanovsky in->lwm = MLX5_GET(wq, wq, lwm);
64f02d0d6eSLeon Romanovsky in->pd = MLX5_GET(wq, wq, pd);
65f02d0d6eSLeon Romanovsky in->db_record = MLX5_GET64(wq, wq, dbr_addr);
66f02d0d6eSLeon Romanovsky }
67f02d0d6eSLeon Romanovsky
get_srqc(void * srqc,struct mlx5_srq_attr * in)68f02d0d6eSLeon Romanovsky static void get_srqc(void *srqc, struct mlx5_srq_attr *in)
69f02d0d6eSLeon Romanovsky {
70f02d0d6eSLeon Romanovsky if (MLX5_GET(srqc, srqc, wq_signature))
71f02d0d6eSLeon Romanovsky in->flags &= MLX5_SRQ_FLAG_WQ_SIG;
72f02d0d6eSLeon Romanovsky in->log_page_size = MLX5_GET(srqc, srqc, log_page_size);
73f02d0d6eSLeon Romanovsky in->wqe_shift = MLX5_GET(srqc, srqc, log_rq_stride);
74f02d0d6eSLeon Romanovsky in->log_size = MLX5_GET(srqc, srqc, log_srq_size);
75f02d0d6eSLeon Romanovsky in->page_offset = MLX5_GET(srqc, srqc, page_offset);
76f02d0d6eSLeon Romanovsky in->lwm = MLX5_GET(srqc, srqc, lwm);
77f02d0d6eSLeon Romanovsky in->pd = MLX5_GET(srqc, srqc, pd);
78f02d0d6eSLeon Romanovsky in->db_record = MLX5_GET64(srqc, srqc, dbr_addr);
79f02d0d6eSLeon Romanovsky }
80f02d0d6eSLeon Romanovsky
mlx5_cmd_get_srq(struct mlx5_ib_dev * dev,u32 srqn)81b4990804SLeon Romanovsky struct mlx5_core_srq *mlx5_cmd_get_srq(struct mlx5_ib_dev *dev, u32 srqn)
82f02d0d6eSLeon Romanovsky {
83f3da6577SLeon Romanovsky struct mlx5_srq_table *table = &dev->srq_table;
84f02d0d6eSLeon Romanovsky struct mlx5_core_srq *srq;
85f02d0d6eSLeon Romanovsky
86c3d6057eSMaor Gottlieb xa_lock_irq(&table->array);
87b02a29ebSMatthew Wilcox srq = xa_load(&table->array, srqn);
88f02d0d6eSLeon Romanovsky if (srq)
8994f3e14eSChuhong Yuan refcount_inc(&srq->common.refcount);
90c3d6057eSMaor Gottlieb xa_unlock_irq(&table->array);
91f02d0d6eSLeon Romanovsky
92f02d0d6eSLeon Romanovsky return srq;
93f02d0d6eSLeon Romanovsky }
94f02d0d6eSLeon Romanovsky
__set_srq_page_size(struct mlx5_srq_attr * in,unsigned long page_size)95*8a7904a6SJason Gunthorpe static int __set_srq_page_size(struct mlx5_srq_attr *in,
96*8a7904a6SJason Gunthorpe unsigned long page_size)
97*8a7904a6SJason Gunthorpe {
98*8a7904a6SJason Gunthorpe if (!page_size)
99*8a7904a6SJason Gunthorpe return -EINVAL;
100*8a7904a6SJason Gunthorpe in->log_page_size = order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT;
101*8a7904a6SJason Gunthorpe
102*8a7904a6SJason Gunthorpe if (WARN_ON(get_pas_size(in) !=
103*8a7904a6SJason Gunthorpe ib_umem_num_dma_blocks(in->umem, page_size) * sizeof(u64)))
104*8a7904a6SJason Gunthorpe return -EINVAL;
105*8a7904a6SJason Gunthorpe return 0;
106*8a7904a6SJason Gunthorpe }
107*8a7904a6SJason Gunthorpe
108*8a7904a6SJason Gunthorpe #define set_srq_page_size(in, typ, log_pgsz_fld) \
109*8a7904a6SJason Gunthorpe __set_srq_page_size(in, mlx5_umem_find_best_quantized_pgoff( \
110*8a7904a6SJason Gunthorpe (in)->umem, typ, log_pgsz_fld, \
111*8a7904a6SJason Gunthorpe MLX5_ADAPTER_PAGE_SHIFT, page_offset, \
112*8a7904a6SJason Gunthorpe 64, &(in)->page_offset))
113*8a7904a6SJason Gunthorpe
create_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * in)114b4990804SLeon Romanovsky static int create_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
115f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *in)
116f02d0d6eSLeon Romanovsky {
117f02d0d6eSLeon Romanovsky u32 create_out[MLX5_ST_SZ_DW(create_srq_out)] = {0};
118f02d0d6eSLeon Romanovsky void *create_in;
119f02d0d6eSLeon Romanovsky void *srqc;
120f02d0d6eSLeon Romanovsky void *pas;
121f02d0d6eSLeon Romanovsky int pas_size;
122f02d0d6eSLeon Romanovsky int inlen;
123f02d0d6eSLeon Romanovsky int err;
124f02d0d6eSLeon Romanovsky
125*8a7904a6SJason Gunthorpe if (in->umem) {
126*8a7904a6SJason Gunthorpe err = set_srq_page_size(in, srqc, log_page_size);
127*8a7904a6SJason Gunthorpe if (err)
128*8a7904a6SJason Gunthorpe return err;
129*8a7904a6SJason Gunthorpe }
130*8a7904a6SJason Gunthorpe
131f02d0d6eSLeon Romanovsky pas_size = get_pas_size(in);
132f02d0d6eSLeon Romanovsky inlen = MLX5_ST_SZ_BYTES(create_srq_in) + pas_size;
133f02d0d6eSLeon Romanovsky create_in = kvzalloc(inlen, GFP_KERNEL);
134f02d0d6eSLeon Romanovsky if (!create_in)
135f02d0d6eSLeon Romanovsky return -ENOMEM;
136f02d0d6eSLeon Romanovsky
137f02d0d6eSLeon Romanovsky MLX5_SET(create_srq_in, create_in, uid, in->uid);
138f02d0d6eSLeon Romanovsky srqc = MLX5_ADDR_OF(create_srq_in, create_in, srq_context_entry);
139f02d0d6eSLeon Romanovsky pas = MLX5_ADDR_OF(create_srq_in, create_in, pas);
140f02d0d6eSLeon Romanovsky
141f02d0d6eSLeon Romanovsky set_srqc(srqc, in);
142*8a7904a6SJason Gunthorpe if (in->umem)
143*8a7904a6SJason Gunthorpe mlx5_ib_populate_pas(
144*8a7904a6SJason Gunthorpe in->umem,
145*8a7904a6SJason Gunthorpe 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
146*8a7904a6SJason Gunthorpe pas, 0);
147*8a7904a6SJason Gunthorpe else
148f02d0d6eSLeon Romanovsky memcpy(pas, in->pas, pas_size);
149f02d0d6eSLeon Romanovsky
150f02d0d6eSLeon Romanovsky MLX5_SET(create_srq_in, create_in, opcode,
151f02d0d6eSLeon Romanovsky MLX5_CMD_OP_CREATE_SRQ);
152f02d0d6eSLeon Romanovsky
153b4990804SLeon Romanovsky err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
154f02d0d6eSLeon Romanovsky sizeof(create_out));
155f02d0d6eSLeon Romanovsky kvfree(create_in);
156f02d0d6eSLeon Romanovsky if (!err) {
157f02d0d6eSLeon Romanovsky srq->srqn = MLX5_GET(create_srq_out, create_out, srqn);
158f02d0d6eSLeon Romanovsky srq->uid = in->uid;
159f02d0d6eSLeon Romanovsky }
160f02d0d6eSLeon Romanovsky
161f02d0d6eSLeon Romanovsky return err;
162f02d0d6eSLeon Romanovsky }
163f02d0d6eSLeon Romanovsky
destroy_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq)164b4990804SLeon Romanovsky static int destroy_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
165f02d0d6eSLeon Romanovsky {
16631578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(destroy_srq_in)] = {};
167f02d0d6eSLeon Romanovsky
16831578defSLeon Romanovsky MLX5_SET(destroy_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_SRQ);
16931578defSLeon Romanovsky MLX5_SET(destroy_srq_in, in, srqn, srq->srqn);
17031578defSLeon Romanovsky MLX5_SET(destroy_srq_in, in, uid, srq->uid);
171f02d0d6eSLeon Romanovsky
17231578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, destroy_srq, in);
173f02d0d6eSLeon Romanovsky }
174f02d0d6eSLeon Romanovsky
arm_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,u16 lwm,int is_srq)175b4990804SLeon Romanovsky static int arm_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
176f02d0d6eSLeon Romanovsky u16 lwm, int is_srq)
177f02d0d6eSLeon Romanovsky {
17831578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
179f02d0d6eSLeon Romanovsky
18031578defSLeon Romanovsky MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
18131578defSLeon Romanovsky MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_SRQ);
18231578defSLeon Romanovsky MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
18331578defSLeon Romanovsky MLX5_SET(arm_rq_in, in, lwm, lwm);
18431578defSLeon Romanovsky MLX5_SET(arm_rq_in, in, uid, srq->uid);
185f02d0d6eSLeon Romanovsky
18631578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
187f02d0d6eSLeon Romanovsky }
188f02d0d6eSLeon Romanovsky
query_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * out)189b4990804SLeon Romanovsky static int query_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
190f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *out)
191f02d0d6eSLeon Romanovsky {
19231578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(query_srq_in)] = {};
193f02d0d6eSLeon Romanovsky u32 *srq_out;
194f02d0d6eSLeon Romanovsky void *srqc;
195f02d0d6eSLeon Romanovsky int err;
196f02d0d6eSLeon Romanovsky
197f02d0d6eSLeon Romanovsky srq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_srq_out), GFP_KERNEL);
198f02d0d6eSLeon Romanovsky if (!srq_out)
199f02d0d6eSLeon Romanovsky return -ENOMEM;
200f02d0d6eSLeon Romanovsky
20131578defSLeon Romanovsky MLX5_SET(query_srq_in, in, opcode, MLX5_CMD_OP_QUERY_SRQ);
20231578defSLeon Romanovsky MLX5_SET(query_srq_in, in, srqn, srq->srqn);
20359dde4d1SLeon Romanovsky err = mlx5_cmd_exec_inout(dev->mdev, query_srq, in, srq_out);
204f02d0d6eSLeon Romanovsky if (err)
205f02d0d6eSLeon Romanovsky goto out;
206f02d0d6eSLeon Romanovsky
20759dde4d1SLeon Romanovsky srqc = MLX5_ADDR_OF(query_srq_out, srq_out, srq_context_entry);
208f02d0d6eSLeon Romanovsky get_srqc(srqc, out);
209f02d0d6eSLeon Romanovsky if (MLX5_GET(srqc, srqc, state) != MLX5_SRQC_STATE_GOOD)
210f02d0d6eSLeon Romanovsky out->flags |= MLX5_SRQ_FLAG_ERR;
211f02d0d6eSLeon Romanovsky out:
21259dde4d1SLeon Romanovsky kvfree(srq_out);
213f02d0d6eSLeon Romanovsky return err;
214f02d0d6eSLeon Romanovsky }
215f02d0d6eSLeon Romanovsky
create_xrc_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * in)216b4990804SLeon Romanovsky static int create_xrc_srq_cmd(struct mlx5_ib_dev *dev,
217f02d0d6eSLeon Romanovsky struct mlx5_core_srq *srq,
218f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *in)
219f02d0d6eSLeon Romanovsky {
220f02d0d6eSLeon Romanovsky u32 create_out[MLX5_ST_SZ_DW(create_xrc_srq_out)];
221f02d0d6eSLeon Romanovsky void *create_in;
222f02d0d6eSLeon Romanovsky void *xrc_srqc;
223f02d0d6eSLeon Romanovsky void *pas;
224f02d0d6eSLeon Romanovsky int pas_size;
225f02d0d6eSLeon Romanovsky int inlen;
226f02d0d6eSLeon Romanovsky int err;
227f02d0d6eSLeon Romanovsky
228*8a7904a6SJason Gunthorpe if (in->umem) {
229*8a7904a6SJason Gunthorpe err = set_srq_page_size(in, xrc_srqc, log_page_size);
230*8a7904a6SJason Gunthorpe if (err)
231*8a7904a6SJason Gunthorpe return err;
232*8a7904a6SJason Gunthorpe }
233*8a7904a6SJason Gunthorpe
234f02d0d6eSLeon Romanovsky pas_size = get_pas_size(in);
235f02d0d6eSLeon Romanovsky inlen = MLX5_ST_SZ_BYTES(create_xrc_srq_in) + pas_size;
236f02d0d6eSLeon Romanovsky create_in = kvzalloc(inlen, GFP_KERNEL);
237f02d0d6eSLeon Romanovsky if (!create_in)
238f02d0d6eSLeon Romanovsky return -ENOMEM;
239f02d0d6eSLeon Romanovsky
240f02d0d6eSLeon Romanovsky MLX5_SET(create_xrc_srq_in, create_in, uid, in->uid);
241f02d0d6eSLeon Romanovsky xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, create_in,
242f02d0d6eSLeon Romanovsky xrc_srq_context_entry);
243f02d0d6eSLeon Romanovsky pas = MLX5_ADDR_OF(create_xrc_srq_in, create_in, pas);
244f02d0d6eSLeon Romanovsky
245f02d0d6eSLeon Romanovsky set_srqc(xrc_srqc, in);
246f02d0d6eSLeon Romanovsky MLX5_SET(xrc_srqc, xrc_srqc, user_index, in->user_index);
247*8a7904a6SJason Gunthorpe if (in->umem)
248*8a7904a6SJason Gunthorpe mlx5_ib_populate_pas(
249*8a7904a6SJason Gunthorpe in->umem,
250*8a7904a6SJason Gunthorpe 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
251*8a7904a6SJason Gunthorpe pas, 0);
252*8a7904a6SJason Gunthorpe else
253f02d0d6eSLeon Romanovsky memcpy(pas, in->pas, pas_size);
254f02d0d6eSLeon Romanovsky MLX5_SET(create_xrc_srq_in, create_in, opcode,
255f02d0d6eSLeon Romanovsky MLX5_CMD_OP_CREATE_XRC_SRQ);
256f02d0d6eSLeon Romanovsky
257f02d0d6eSLeon Romanovsky memset(create_out, 0, sizeof(create_out));
258b4990804SLeon Romanovsky err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
259f02d0d6eSLeon Romanovsky sizeof(create_out));
260f02d0d6eSLeon Romanovsky if (err)
261f02d0d6eSLeon Romanovsky goto out;
262f02d0d6eSLeon Romanovsky
263f02d0d6eSLeon Romanovsky srq->srqn = MLX5_GET(create_xrc_srq_out, create_out, xrc_srqn);
264f02d0d6eSLeon Romanovsky srq->uid = in->uid;
265f02d0d6eSLeon Romanovsky out:
266f02d0d6eSLeon Romanovsky kvfree(create_in);
267f02d0d6eSLeon Romanovsky return err;
268f02d0d6eSLeon Romanovsky }
269f02d0d6eSLeon Romanovsky
destroy_xrc_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq)270b4990804SLeon Romanovsky static int destroy_xrc_srq_cmd(struct mlx5_ib_dev *dev,
271f02d0d6eSLeon Romanovsky struct mlx5_core_srq *srq)
272f02d0d6eSLeon Romanovsky {
27331578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(destroy_xrc_srq_in)] = {};
274f02d0d6eSLeon Romanovsky
27531578defSLeon Romanovsky MLX5_SET(destroy_xrc_srq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRC_SRQ);
27631578defSLeon Romanovsky MLX5_SET(destroy_xrc_srq_in, in, xrc_srqn, srq->srqn);
27731578defSLeon Romanovsky MLX5_SET(destroy_xrc_srq_in, in, uid, srq->uid);
278f02d0d6eSLeon Romanovsky
27931578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, destroy_xrc_srq, in);
280f02d0d6eSLeon Romanovsky }
281f02d0d6eSLeon Romanovsky
arm_xrc_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,u16 lwm)282b4990804SLeon Romanovsky static int arm_xrc_srq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
283b4990804SLeon Romanovsky u16 lwm)
284f02d0d6eSLeon Romanovsky {
28531578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(arm_xrc_srq_in)] = {};
286f02d0d6eSLeon Romanovsky
28731578defSLeon Romanovsky MLX5_SET(arm_xrc_srq_in, in, opcode, MLX5_CMD_OP_ARM_XRC_SRQ);
28831578defSLeon Romanovsky MLX5_SET(arm_xrc_srq_in, in, op_mod,
28931578defSLeon Romanovsky MLX5_ARM_XRC_SRQ_IN_OP_MOD_XRC_SRQ);
29031578defSLeon Romanovsky MLX5_SET(arm_xrc_srq_in, in, xrc_srqn, srq->srqn);
29131578defSLeon Romanovsky MLX5_SET(arm_xrc_srq_in, in, lwm, lwm);
29231578defSLeon Romanovsky MLX5_SET(arm_xrc_srq_in, in, uid, srq->uid);
293f02d0d6eSLeon Romanovsky
29431578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, arm_xrc_srq, in);
295f02d0d6eSLeon Romanovsky }
296f02d0d6eSLeon Romanovsky
query_xrc_srq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * out)297b4990804SLeon Romanovsky static int query_xrc_srq_cmd(struct mlx5_ib_dev *dev,
298f02d0d6eSLeon Romanovsky struct mlx5_core_srq *srq,
299f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *out)
300f02d0d6eSLeon Romanovsky {
30131578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(query_xrc_srq_in)] = {};
302f02d0d6eSLeon Romanovsky u32 *xrcsrq_out;
303f02d0d6eSLeon Romanovsky void *xrc_srqc;
304f02d0d6eSLeon Romanovsky int err;
305f02d0d6eSLeon Romanovsky
306f02d0d6eSLeon Romanovsky xrcsrq_out = kvzalloc(MLX5_ST_SZ_BYTES(query_xrc_srq_out), GFP_KERNEL);
307f02d0d6eSLeon Romanovsky if (!xrcsrq_out)
308f02d0d6eSLeon Romanovsky return -ENOMEM;
309f02d0d6eSLeon Romanovsky
31031578defSLeon Romanovsky MLX5_SET(query_xrc_srq_in, in, opcode, MLX5_CMD_OP_QUERY_XRC_SRQ);
31131578defSLeon Romanovsky MLX5_SET(query_xrc_srq_in, in, xrc_srqn, srq->srqn);
312f02d0d6eSLeon Romanovsky
31331578defSLeon Romanovsky err = mlx5_cmd_exec_inout(dev->mdev, query_xrc_srq, in, xrcsrq_out);
314f02d0d6eSLeon Romanovsky if (err)
315f02d0d6eSLeon Romanovsky goto out;
316f02d0d6eSLeon Romanovsky
317f02d0d6eSLeon Romanovsky xrc_srqc = MLX5_ADDR_OF(query_xrc_srq_out, xrcsrq_out,
318f02d0d6eSLeon Romanovsky xrc_srq_context_entry);
319f02d0d6eSLeon Romanovsky get_srqc(xrc_srqc, out);
320f02d0d6eSLeon Romanovsky if (MLX5_GET(xrc_srqc, xrc_srqc, state) != MLX5_XRC_SRQC_STATE_GOOD)
321f02d0d6eSLeon Romanovsky out->flags |= MLX5_SRQ_FLAG_ERR;
322f02d0d6eSLeon Romanovsky
323f02d0d6eSLeon Romanovsky out:
324f02d0d6eSLeon Romanovsky kvfree(xrcsrq_out);
325f02d0d6eSLeon Romanovsky return err;
326f02d0d6eSLeon Romanovsky }
327f02d0d6eSLeon Romanovsky
create_rmp_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * in)328b4990804SLeon Romanovsky static int create_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
329f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *in)
330f02d0d6eSLeon Romanovsky {
331a1eb1802SLeon Romanovsky void *create_out = NULL;
332a1eb1802SLeon Romanovsky void *create_in = NULL;
333f02d0d6eSLeon Romanovsky void *rmpc;
334f02d0d6eSLeon Romanovsky void *wq;
335*8a7904a6SJason Gunthorpe void *pas;
336f02d0d6eSLeon Romanovsky int pas_size;
337a1eb1802SLeon Romanovsky int outlen;
338f02d0d6eSLeon Romanovsky int inlen;
339f02d0d6eSLeon Romanovsky int err;
340f02d0d6eSLeon Romanovsky
341*8a7904a6SJason Gunthorpe if (in->umem) {
342*8a7904a6SJason Gunthorpe err = set_srq_page_size(in, wq, log_wq_pg_sz);
343*8a7904a6SJason Gunthorpe if (err)
344*8a7904a6SJason Gunthorpe return err;
345*8a7904a6SJason Gunthorpe }
346*8a7904a6SJason Gunthorpe
347f02d0d6eSLeon Romanovsky pas_size = get_pas_size(in);
348f02d0d6eSLeon Romanovsky inlen = MLX5_ST_SZ_BYTES(create_rmp_in) + pas_size;
349a1eb1802SLeon Romanovsky outlen = MLX5_ST_SZ_BYTES(create_rmp_out);
350f02d0d6eSLeon Romanovsky create_in = kvzalloc(inlen, GFP_KERNEL);
351a1eb1802SLeon Romanovsky create_out = kvzalloc(outlen, GFP_KERNEL);
352a1eb1802SLeon Romanovsky if (!create_in || !create_out) {
353a1eb1802SLeon Romanovsky err = -ENOMEM;
354a1eb1802SLeon Romanovsky goto out;
355a1eb1802SLeon Romanovsky }
356f02d0d6eSLeon Romanovsky
357f02d0d6eSLeon Romanovsky rmpc = MLX5_ADDR_OF(create_rmp_in, create_in, ctx);
358f02d0d6eSLeon Romanovsky wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
359f02d0d6eSLeon Romanovsky
360f02d0d6eSLeon Romanovsky MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
361f02d0d6eSLeon Romanovsky MLX5_SET(create_rmp_in, create_in, uid, in->uid);
362*8a7904a6SJason Gunthorpe pas = MLX5_ADDR_OF(rmpc, rmpc, wq.pas);
363*8a7904a6SJason Gunthorpe
364f02d0d6eSLeon Romanovsky set_wq(wq, in);
365*8a7904a6SJason Gunthorpe if (in->umem)
366*8a7904a6SJason Gunthorpe mlx5_ib_populate_pas(
367*8a7904a6SJason Gunthorpe in->umem,
368*8a7904a6SJason Gunthorpe 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
369*8a7904a6SJason Gunthorpe pas, 0);
370*8a7904a6SJason Gunthorpe else
371*8a7904a6SJason Gunthorpe memcpy(pas, in->pas, pas_size);
372f02d0d6eSLeon Romanovsky
373a1eb1802SLeon Romanovsky MLX5_SET(create_rmp_in, create_in, opcode, MLX5_CMD_OP_CREATE_RMP);
374a1eb1802SLeon Romanovsky err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out, outlen);
375a1eb1802SLeon Romanovsky if (!err) {
376a1eb1802SLeon Romanovsky srq->srqn = MLX5_GET(create_rmp_out, create_out, rmpn);
377f02d0d6eSLeon Romanovsky srq->uid = in->uid;
378a1eb1802SLeon Romanovsky }
379f02d0d6eSLeon Romanovsky
380a1eb1802SLeon Romanovsky out:
381f02d0d6eSLeon Romanovsky kvfree(create_in);
382a1eb1802SLeon Romanovsky kvfree(create_out);
383f02d0d6eSLeon Romanovsky return err;
384f02d0d6eSLeon Romanovsky }
385f02d0d6eSLeon Romanovsky
destroy_rmp_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq)386b4990804SLeon Romanovsky static int destroy_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
387f02d0d6eSLeon Romanovsky {
388f02d0d6eSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(destroy_rmp_in)] = {};
389f02d0d6eSLeon Romanovsky
390f02d0d6eSLeon Romanovsky MLX5_SET(destroy_rmp_in, in, opcode, MLX5_CMD_OP_DESTROY_RMP);
391f02d0d6eSLeon Romanovsky MLX5_SET(destroy_rmp_in, in, rmpn, srq->srqn);
392f02d0d6eSLeon Romanovsky MLX5_SET(destroy_rmp_in, in, uid, srq->uid);
39331578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, destroy_rmp, in);
394f02d0d6eSLeon Romanovsky }
395f02d0d6eSLeon Romanovsky
arm_rmp_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,u16 lwm)396b4990804SLeon Romanovsky static int arm_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
397f02d0d6eSLeon Romanovsky u16 lwm)
398f02d0d6eSLeon Romanovsky {
39936ff4880SLeon Romanovsky void *out = NULL;
40036ff4880SLeon Romanovsky void *in = NULL;
401f02d0d6eSLeon Romanovsky void *rmpc;
402f02d0d6eSLeon Romanovsky void *wq;
403f02d0d6eSLeon Romanovsky void *bitmask;
40436ff4880SLeon Romanovsky int outlen;
40536ff4880SLeon Romanovsky int inlen;
406f02d0d6eSLeon Romanovsky int err;
407f02d0d6eSLeon Romanovsky
40836ff4880SLeon Romanovsky inlen = MLX5_ST_SZ_BYTES(modify_rmp_in);
40936ff4880SLeon Romanovsky outlen = MLX5_ST_SZ_BYTES(modify_rmp_out);
41036ff4880SLeon Romanovsky
41136ff4880SLeon Romanovsky in = kvzalloc(inlen, GFP_KERNEL);
41236ff4880SLeon Romanovsky out = kvzalloc(outlen, GFP_KERNEL);
41336ff4880SLeon Romanovsky if (!in || !out) {
41436ff4880SLeon Romanovsky err = -ENOMEM;
41536ff4880SLeon Romanovsky goto out;
41636ff4880SLeon Romanovsky }
417f02d0d6eSLeon Romanovsky
418f02d0d6eSLeon Romanovsky rmpc = MLX5_ADDR_OF(modify_rmp_in, in, ctx);
419f02d0d6eSLeon Romanovsky bitmask = MLX5_ADDR_OF(modify_rmp_in, in, bitmask);
420f02d0d6eSLeon Romanovsky wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
421f02d0d6eSLeon Romanovsky
422f02d0d6eSLeon Romanovsky MLX5_SET(modify_rmp_in, in, rmp_state, MLX5_RMPC_STATE_RDY);
423f02d0d6eSLeon Romanovsky MLX5_SET(modify_rmp_in, in, rmpn, srq->srqn);
424f02d0d6eSLeon Romanovsky MLX5_SET(modify_rmp_in, in, uid, srq->uid);
425f02d0d6eSLeon Romanovsky MLX5_SET(wq, wq, lwm, lwm);
426f02d0d6eSLeon Romanovsky MLX5_SET(rmp_bitmask, bitmask, lwm, 1);
427f02d0d6eSLeon Romanovsky MLX5_SET(rmpc, rmpc, state, MLX5_RMPC_STATE_RDY);
42836ff4880SLeon Romanovsky MLX5_SET(modify_rmp_in, in, opcode, MLX5_CMD_OP_MODIFY_RMP);
429f02d0d6eSLeon Romanovsky
43031578defSLeon Romanovsky err = mlx5_cmd_exec_inout(dev->mdev, modify_rmp, in, out);
431f02d0d6eSLeon Romanovsky
43236ff4880SLeon Romanovsky out:
433f02d0d6eSLeon Romanovsky kvfree(in);
43436ff4880SLeon Romanovsky kvfree(out);
435f02d0d6eSLeon Romanovsky return err;
436f02d0d6eSLeon Romanovsky }
437f02d0d6eSLeon Romanovsky
query_rmp_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * out)438b4990804SLeon Romanovsky static int query_rmp_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
439f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *out)
440f02d0d6eSLeon Romanovsky {
44136ff4880SLeon Romanovsky u32 *rmp_out = NULL;
44236ff4880SLeon Romanovsky u32 *rmp_in = NULL;
443f02d0d6eSLeon Romanovsky void *rmpc;
44436ff4880SLeon Romanovsky int outlen;
44536ff4880SLeon Romanovsky int inlen;
446f02d0d6eSLeon Romanovsky int err;
447f02d0d6eSLeon Romanovsky
44836ff4880SLeon Romanovsky outlen = MLX5_ST_SZ_BYTES(query_rmp_out);
44936ff4880SLeon Romanovsky inlen = MLX5_ST_SZ_BYTES(query_rmp_in);
450f02d0d6eSLeon Romanovsky
45136ff4880SLeon Romanovsky rmp_out = kvzalloc(outlen, GFP_KERNEL);
45236ff4880SLeon Romanovsky rmp_in = kvzalloc(inlen, GFP_KERNEL);
45336ff4880SLeon Romanovsky if (!rmp_out || !rmp_in) {
45436ff4880SLeon Romanovsky err = -ENOMEM;
45536ff4880SLeon Romanovsky goto out;
45636ff4880SLeon Romanovsky }
45736ff4880SLeon Romanovsky
45836ff4880SLeon Romanovsky MLX5_SET(query_rmp_in, rmp_in, opcode, MLX5_CMD_OP_QUERY_RMP);
45936ff4880SLeon Romanovsky MLX5_SET(query_rmp_in, rmp_in, rmpn, srq->srqn);
46031578defSLeon Romanovsky err = mlx5_cmd_exec_inout(dev->mdev, query_rmp, rmp_in, rmp_out);
461f02d0d6eSLeon Romanovsky if (err)
462f02d0d6eSLeon Romanovsky goto out;
463f02d0d6eSLeon Romanovsky
464f02d0d6eSLeon Romanovsky rmpc = MLX5_ADDR_OF(query_rmp_out, rmp_out, rmp_context);
465f02d0d6eSLeon Romanovsky get_wq(MLX5_ADDR_OF(rmpc, rmpc, wq), out);
466f02d0d6eSLeon Romanovsky if (MLX5_GET(rmpc, rmpc, state) != MLX5_RMPC_STATE_RDY)
467f02d0d6eSLeon Romanovsky out->flags |= MLX5_SRQ_FLAG_ERR;
468f02d0d6eSLeon Romanovsky
469f02d0d6eSLeon Romanovsky out:
470f02d0d6eSLeon Romanovsky kvfree(rmp_out);
47136ff4880SLeon Romanovsky kvfree(rmp_in);
472f02d0d6eSLeon Romanovsky return err;
473f02d0d6eSLeon Romanovsky }
474f02d0d6eSLeon Romanovsky
create_xrq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * in)475b4990804SLeon Romanovsky static int create_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
476f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *in)
477f02d0d6eSLeon Romanovsky {
478f02d0d6eSLeon Romanovsky u32 create_out[MLX5_ST_SZ_DW(create_xrq_out)] = {0};
479f02d0d6eSLeon Romanovsky void *create_in;
480f02d0d6eSLeon Romanovsky void *xrqc;
481f02d0d6eSLeon Romanovsky void *wq;
482*8a7904a6SJason Gunthorpe void *pas;
483f02d0d6eSLeon Romanovsky int pas_size;
484f02d0d6eSLeon Romanovsky int inlen;
485f02d0d6eSLeon Romanovsky int err;
486f02d0d6eSLeon Romanovsky
487*8a7904a6SJason Gunthorpe if (in->umem) {
488*8a7904a6SJason Gunthorpe err = set_srq_page_size(in, wq, log_wq_pg_sz);
489*8a7904a6SJason Gunthorpe if (err)
490*8a7904a6SJason Gunthorpe return err;
491*8a7904a6SJason Gunthorpe }
492*8a7904a6SJason Gunthorpe
493f02d0d6eSLeon Romanovsky pas_size = get_pas_size(in);
494f02d0d6eSLeon Romanovsky inlen = MLX5_ST_SZ_BYTES(create_xrq_in) + pas_size;
495f02d0d6eSLeon Romanovsky create_in = kvzalloc(inlen, GFP_KERNEL);
496f02d0d6eSLeon Romanovsky if (!create_in)
497f02d0d6eSLeon Romanovsky return -ENOMEM;
498f02d0d6eSLeon Romanovsky
499f02d0d6eSLeon Romanovsky xrqc = MLX5_ADDR_OF(create_xrq_in, create_in, xrq_context);
500f02d0d6eSLeon Romanovsky wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
501*8a7904a6SJason Gunthorpe pas = MLX5_ADDR_OF(xrqc, xrqc, wq.pas);
502f02d0d6eSLeon Romanovsky
503f02d0d6eSLeon Romanovsky set_wq(wq, in);
504*8a7904a6SJason Gunthorpe if (in->umem)
505*8a7904a6SJason Gunthorpe mlx5_ib_populate_pas(
506*8a7904a6SJason Gunthorpe in->umem,
507*8a7904a6SJason Gunthorpe 1UL << (in->log_page_size + MLX5_ADAPTER_PAGE_SHIFT),
508*8a7904a6SJason Gunthorpe pas, 0);
509*8a7904a6SJason Gunthorpe else
510*8a7904a6SJason Gunthorpe memcpy(pas, in->pas, pas_size);
511f02d0d6eSLeon Romanovsky
512f02d0d6eSLeon Romanovsky if (in->type == IB_SRQT_TM) {
513f02d0d6eSLeon Romanovsky MLX5_SET(xrqc, xrqc, topology, MLX5_XRQC_TOPOLOGY_TAG_MATCHING);
514f02d0d6eSLeon Romanovsky if (in->flags & MLX5_SRQ_FLAG_RNDV)
515f02d0d6eSLeon Romanovsky MLX5_SET(xrqc, xrqc, offload, MLX5_XRQC_OFFLOAD_RNDV);
516f02d0d6eSLeon Romanovsky MLX5_SET(xrqc, xrqc,
517f02d0d6eSLeon Romanovsky tag_matching_topology_context.log_matching_list_sz,
518f02d0d6eSLeon Romanovsky in->tm_log_list_size);
519f02d0d6eSLeon Romanovsky }
520f02d0d6eSLeon Romanovsky MLX5_SET(xrqc, xrqc, user_index, in->user_index);
521f02d0d6eSLeon Romanovsky MLX5_SET(xrqc, xrqc, cqn, in->cqn);
522f02d0d6eSLeon Romanovsky MLX5_SET(create_xrq_in, create_in, opcode, MLX5_CMD_OP_CREATE_XRQ);
523f02d0d6eSLeon Romanovsky MLX5_SET(create_xrq_in, create_in, uid, in->uid);
524b4990804SLeon Romanovsky err = mlx5_cmd_exec(dev->mdev, create_in, inlen, create_out,
525f02d0d6eSLeon Romanovsky sizeof(create_out));
526f02d0d6eSLeon Romanovsky kvfree(create_in);
527f02d0d6eSLeon Romanovsky if (!err) {
528f02d0d6eSLeon Romanovsky srq->srqn = MLX5_GET(create_xrq_out, create_out, xrqn);
529f02d0d6eSLeon Romanovsky srq->uid = in->uid;
530f02d0d6eSLeon Romanovsky }
531f02d0d6eSLeon Romanovsky
532f02d0d6eSLeon Romanovsky return err;
533f02d0d6eSLeon Romanovsky }
534f02d0d6eSLeon Romanovsky
destroy_xrq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq)535b4990804SLeon Romanovsky static int destroy_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
536f02d0d6eSLeon Romanovsky {
53731578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(destroy_xrq_in)] = {};
538f02d0d6eSLeon Romanovsky
539f02d0d6eSLeon Romanovsky MLX5_SET(destroy_xrq_in, in, opcode, MLX5_CMD_OP_DESTROY_XRQ);
540f02d0d6eSLeon Romanovsky MLX5_SET(destroy_xrq_in, in, xrqn, srq->srqn);
541f02d0d6eSLeon Romanovsky MLX5_SET(destroy_xrq_in, in, uid, srq->uid);
542f02d0d6eSLeon Romanovsky
54331578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, destroy_xrq, in);
544f02d0d6eSLeon Romanovsky }
545f02d0d6eSLeon Romanovsky
arm_xrq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,u16 lwm)546b4990804SLeon Romanovsky static int arm_xrq_cmd(struct mlx5_ib_dev *dev,
547f02d0d6eSLeon Romanovsky struct mlx5_core_srq *srq,
548f02d0d6eSLeon Romanovsky u16 lwm)
549f02d0d6eSLeon Romanovsky {
55031578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(arm_rq_in)] = {};
551f02d0d6eSLeon Romanovsky
552f02d0d6eSLeon Romanovsky MLX5_SET(arm_rq_in, in, opcode, MLX5_CMD_OP_ARM_RQ);
553f02d0d6eSLeon Romanovsky MLX5_SET(arm_rq_in, in, op_mod, MLX5_ARM_RQ_IN_OP_MOD_XRQ);
554f02d0d6eSLeon Romanovsky MLX5_SET(arm_rq_in, in, srq_number, srq->srqn);
555f02d0d6eSLeon Romanovsky MLX5_SET(arm_rq_in, in, lwm, lwm);
556f02d0d6eSLeon Romanovsky MLX5_SET(arm_rq_in, in, uid, srq->uid);
557f02d0d6eSLeon Romanovsky
55831578defSLeon Romanovsky return mlx5_cmd_exec_in(dev->mdev, arm_rq, in);
559f02d0d6eSLeon Romanovsky }
560f02d0d6eSLeon Romanovsky
query_xrq_cmd(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * out)561b4990804SLeon Romanovsky static int query_xrq_cmd(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
562f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *out)
563f02d0d6eSLeon Romanovsky {
56431578defSLeon Romanovsky u32 in[MLX5_ST_SZ_DW(query_xrq_in)] = {};
565f02d0d6eSLeon Romanovsky u32 *xrq_out;
566f02d0d6eSLeon Romanovsky int outlen = MLX5_ST_SZ_BYTES(query_xrq_out);
567f02d0d6eSLeon Romanovsky void *xrqc;
568f02d0d6eSLeon Romanovsky int err;
569f02d0d6eSLeon Romanovsky
570f02d0d6eSLeon Romanovsky xrq_out = kvzalloc(outlen, GFP_KERNEL);
571f02d0d6eSLeon Romanovsky if (!xrq_out)
572f02d0d6eSLeon Romanovsky return -ENOMEM;
573f02d0d6eSLeon Romanovsky
574f02d0d6eSLeon Romanovsky MLX5_SET(query_xrq_in, in, opcode, MLX5_CMD_OP_QUERY_XRQ);
575f02d0d6eSLeon Romanovsky MLX5_SET(query_xrq_in, in, xrqn, srq->srqn);
576f02d0d6eSLeon Romanovsky
57731578defSLeon Romanovsky err = mlx5_cmd_exec_inout(dev->mdev, query_xrq, in, xrq_out);
578f02d0d6eSLeon Romanovsky if (err)
579f02d0d6eSLeon Romanovsky goto out;
580f02d0d6eSLeon Romanovsky
581f02d0d6eSLeon Romanovsky xrqc = MLX5_ADDR_OF(query_xrq_out, xrq_out, xrq_context);
582f02d0d6eSLeon Romanovsky get_wq(MLX5_ADDR_OF(xrqc, xrqc, wq), out);
583f02d0d6eSLeon Romanovsky if (MLX5_GET(xrqc, xrqc, state) != MLX5_XRQC_STATE_GOOD)
584f02d0d6eSLeon Romanovsky out->flags |= MLX5_SRQ_FLAG_ERR;
585f02d0d6eSLeon Romanovsky out->tm_next_tag =
586f02d0d6eSLeon Romanovsky MLX5_GET(xrqc, xrqc,
587f02d0d6eSLeon Romanovsky tag_matching_topology_context.append_next_index);
588f02d0d6eSLeon Romanovsky out->tm_hw_phase_cnt =
589f02d0d6eSLeon Romanovsky MLX5_GET(xrqc, xrqc,
590f02d0d6eSLeon Romanovsky tag_matching_topology_context.hw_phase_cnt);
591f02d0d6eSLeon Romanovsky out->tm_sw_phase_cnt =
592f02d0d6eSLeon Romanovsky MLX5_GET(xrqc, xrqc,
593f02d0d6eSLeon Romanovsky tag_matching_topology_context.sw_phase_cnt);
594f02d0d6eSLeon Romanovsky
595f02d0d6eSLeon Romanovsky out:
596f02d0d6eSLeon Romanovsky kvfree(xrq_out);
597f02d0d6eSLeon Romanovsky return err;
598f02d0d6eSLeon Romanovsky }
599f02d0d6eSLeon Romanovsky
create_srq_split(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * in)600b4990804SLeon Romanovsky static int create_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
601f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *in)
602f02d0d6eSLeon Romanovsky {
603b4990804SLeon Romanovsky if (!dev->mdev->issi)
604f02d0d6eSLeon Romanovsky return create_srq_cmd(dev, srq, in);
605f02d0d6eSLeon Romanovsky switch (srq->common.res) {
606f02d0d6eSLeon Romanovsky case MLX5_RES_XSRQ:
607f02d0d6eSLeon Romanovsky return create_xrc_srq_cmd(dev, srq, in);
608f02d0d6eSLeon Romanovsky case MLX5_RES_XRQ:
609f02d0d6eSLeon Romanovsky return create_xrq_cmd(dev, srq, in);
610f02d0d6eSLeon Romanovsky default:
611f02d0d6eSLeon Romanovsky return create_rmp_cmd(dev, srq, in);
612f02d0d6eSLeon Romanovsky }
613f02d0d6eSLeon Romanovsky }
614f02d0d6eSLeon Romanovsky
destroy_srq_split(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq)615b4990804SLeon Romanovsky static int destroy_srq_split(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
616f02d0d6eSLeon Romanovsky {
617b4990804SLeon Romanovsky if (!dev->mdev->issi)
618f02d0d6eSLeon Romanovsky return destroy_srq_cmd(dev, srq);
619f02d0d6eSLeon Romanovsky switch (srq->common.res) {
620f02d0d6eSLeon Romanovsky case MLX5_RES_XSRQ:
621f02d0d6eSLeon Romanovsky return destroy_xrc_srq_cmd(dev, srq);
622f02d0d6eSLeon Romanovsky case MLX5_RES_XRQ:
623f02d0d6eSLeon Romanovsky return destroy_xrq_cmd(dev, srq);
624f02d0d6eSLeon Romanovsky default:
625f02d0d6eSLeon Romanovsky return destroy_rmp_cmd(dev, srq);
626f02d0d6eSLeon Romanovsky }
627f02d0d6eSLeon Romanovsky }
628f02d0d6eSLeon Romanovsky
mlx5_cmd_create_srq(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * in)629b4990804SLeon Romanovsky int mlx5_cmd_create_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
630f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *in)
631f02d0d6eSLeon Romanovsky {
632f3da6577SLeon Romanovsky struct mlx5_srq_table *table = &dev->srq_table;
633f02d0d6eSLeon Romanovsky int err;
634b4990804SLeon Romanovsky
635f02d0d6eSLeon Romanovsky switch (in->type) {
636f02d0d6eSLeon Romanovsky case IB_SRQT_XRC:
637f02d0d6eSLeon Romanovsky srq->common.res = MLX5_RES_XSRQ;
638f02d0d6eSLeon Romanovsky break;
639f02d0d6eSLeon Romanovsky case IB_SRQT_TM:
640f02d0d6eSLeon Romanovsky srq->common.res = MLX5_RES_XRQ;
641f02d0d6eSLeon Romanovsky break;
642f02d0d6eSLeon Romanovsky default:
643f02d0d6eSLeon Romanovsky srq->common.res = MLX5_RES_SRQ;
644f02d0d6eSLeon Romanovsky }
645f02d0d6eSLeon Romanovsky
646f02d0d6eSLeon Romanovsky err = create_srq_split(dev, srq, in);
647f02d0d6eSLeon Romanovsky if (err)
648f02d0d6eSLeon Romanovsky return err;
649f02d0d6eSLeon Romanovsky
65094f3e14eSChuhong Yuan refcount_set(&srq->common.refcount, 1);
65110f56242SMoni Shoua init_completion(&srq->common.free);
652f02d0d6eSLeon Romanovsky
653b02a29ebSMatthew Wilcox err = xa_err(xa_store_irq(&table->array, srq->srqn, srq, GFP_KERNEL));
654f02d0d6eSLeon Romanovsky if (err)
655f02d0d6eSLeon Romanovsky goto err_destroy_srq_split;
656f02d0d6eSLeon Romanovsky
657f02d0d6eSLeon Romanovsky return 0;
658f02d0d6eSLeon Romanovsky
659f02d0d6eSLeon Romanovsky err_destroy_srq_split:
660f02d0d6eSLeon Romanovsky destroy_srq_split(dev, srq);
661f02d0d6eSLeon Romanovsky
662f02d0d6eSLeon Romanovsky return err;
663f02d0d6eSLeon Romanovsky }
664f02d0d6eSLeon Romanovsky
mlx5_cmd_destroy_srq(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq)665119181d1SLeon Romanovsky int mlx5_cmd_destroy_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq)
666f02d0d6eSLeon Romanovsky {
667f3da6577SLeon Romanovsky struct mlx5_srq_table *table = &dev->srq_table;
668f02d0d6eSLeon Romanovsky struct mlx5_core_srq *tmp;
669f02d0d6eSLeon Romanovsky int err;
670f02d0d6eSLeon Romanovsky
671fd89099dSLeon Romanovsky /* Delete entry, but leave index occupied */
672fd89099dSLeon Romanovsky tmp = xa_cmpxchg_irq(&table->array, srq->srqn, srq, XA_ZERO_ENTRY, 0);
673fd89099dSLeon Romanovsky if (WARN_ON(tmp != srq))
674119181d1SLeon Romanovsky return xa_err(tmp) ?: -EINVAL;
675f02d0d6eSLeon Romanovsky
676f02d0d6eSLeon Romanovsky err = destroy_srq_split(dev, srq);
677fd89099dSLeon Romanovsky if (err) {
678fd89099dSLeon Romanovsky /*
679fd89099dSLeon Romanovsky * We don't need to check returned result for an error,
680fd89099dSLeon Romanovsky * because we are storing in pre-allocated space xarray
681fd89099dSLeon Romanovsky * entry and it can't fail at this stage.
682fd89099dSLeon Romanovsky */
683fd89099dSLeon Romanovsky xa_cmpxchg_irq(&table->array, srq->srqn, XA_ZERO_ENTRY, srq, 0);
684119181d1SLeon Romanovsky return err;
685fd89099dSLeon Romanovsky }
686fd89099dSLeon Romanovsky xa_erase_irq(&table->array, srq->srqn);
687f02d0d6eSLeon Romanovsky
68810f56242SMoni Shoua mlx5_core_res_put(&srq->common);
68910f56242SMoni Shoua wait_for_completion(&srq->common.free);
690119181d1SLeon Romanovsky return 0;
691f02d0d6eSLeon Romanovsky }
692f02d0d6eSLeon Romanovsky
mlx5_cmd_query_srq(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,struct mlx5_srq_attr * out)693b4990804SLeon Romanovsky int mlx5_cmd_query_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
694f02d0d6eSLeon Romanovsky struct mlx5_srq_attr *out)
695f02d0d6eSLeon Romanovsky {
696b4990804SLeon Romanovsky if (!dev->mdev->issi)
697f02d0d6eSLeon Romanovsky return query_srq_cmd(dev, srq, out);
698f02d0d6eSLeon Romanovsky switch (srq->common.res) {
699f02d0d6eSLeon Romanovsky case MLX5_RES_XSRQ:
700f02d0d6eSLeon Romanovsky return query_xrc_srq_cmd(dev, srq, out);
701f02d0d6eSLeon Romanovsky case MLX5_RES_XRQ:
702f02d0d6eSLeon Romanovsky return query_xrq_cmd(dev, srq, out);
703f02d0d6eSLeon Romanovsky default:
704f02d0d6eSLeon Romanovsky return query_rmp_cmd(dev, srq, out);
705f02d0d6eSLeon Romanovsky }
706f02d0d6eSLeon Romanovsky }
707f02d0d6eSLeon Romanovsky
mlx5_cmd_arm_srq(struct mlx5_ib_dev * dev,struct mlx5_core_srq * srq,u16 lwm,int is_srq)708b4990804SLeon Romanovsky int mlx5_cmd_arm_srq(struct mlx5_ib_dev *dev, struct mlx5_core_srq *srq,
709f02d0d6eSLeon Romanovsky u16 lwm, int is_srq)
710f02d0d6eSLeon Romanovsky {
711b4990804SLeon Romanovsky if (!dev->mdev->issi)
712f02d0d6eSLeon Romanovsky return arm_srq_cmd(dev, srq, lwm, is_srq);
713f02d0d6eSLeon Romanovsky switch (srq->common.res) {
714f02d0d6eSLeon Romanovsky case MLX5_RES_XSRQ:
715f02d0d6eSLeon Romanovsky return arm_xrc_srq_cmd(dev, srq, lwm);
716f02d0d6eSLeon Romanovsky case MLX5_RES_XRQ:
717f02d0d6eSLeon Romanovsky return arm_xrq_cmd(dev, srq, lwm);
718f02d0d6eSLeon Romanovsky default:
719f02d0d6eSLeon Romanovsky return arm_rmp_cmd(dev, srq, lwm);
720f02d0d6eSLeon Romanovsky }
721f02d0d6eSLeon Romanovsky }
722f3da6577SLeon Romanovsky
srq_event_notifier(struct notifier_block * nb,unsigned long type,void * data)723f3da6577SLeon Romanovsky static int srq_event_notifier(struct notifier_block *nb,
724f3da6577SLeon Romanovsky unsigned long type, void *data)
725f3da6577SLeon Romanovsky {
726f3da6577SLeon Romanovsky struct mlx5_srq_table *table;
727f3da6577SLeon Romanovsky struct mlx5_core_srq *srq;
728f3da6577SLeon Romanovsky struct mlx5_eqe *eqe;
729f3da6577SLeon Romanovsky u32 srqn;
730f3da6577SLeon Romanovsky
731f3da6577SLeon Romanovsky if (type != MLX5_EVENT_TYPE_SRQ_CATAS_ERROR &&
732f3da6577SLeon Romanovsky type != MLX5_EVENT_TYPE_SRQ_RQ_LIMIT)
733f3da6577SLeon Romanovsky return NOTIFY_DONE;
734f3da6577SLeon Romanovsky
735f3da6577SLeon Romanovsky table = container_of(nb, struct mlx5_srq_table, nb);
736f3da6577SLeon Romanovsky
737f3da6577SLeon Romanovsky eqe = data;
738f3da6577SLeon Romanovsky srqn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
739f3da6577SLeon Romanovsky
740b02a29ebSMatthew Wilcox xa_lock(&table->array);
741b02a29ebSMatthew Wilcox srq = xa_load(&table->array, srqn);
742f3da6577SLeon Romanovsky if (srq)
74394f3e14eSChuhong Yuan refcount_inc(&srq->common.refcount);
744b02a29ebSMatthew Wilcox xa_unlock(&table->array);
745f3da6577SLeon Romanovsky
746f3da6577SLeon Romanovsky if (!srq)
747f3da6577SLeon Romanovsky return NOTIFY_OK;
748f3da6577SLeon Romanovsky
749f3da6577SLeon Romanovsky srq->event(srq, eqe->type);
750f3da6577SLeon Romanovsky
75110f56242SMoni Shoua mlx5_core_res_put(&srq->common);
752f3da6577SLeon Romanovsky
753f3da6577SLeon Romanovsky return NOTIFY_OK;
754f3da6577SLeon Romanovsky }
755f3da6577SLeon Romanovsky
mlx5_init_srq_table(struct mlx5_ib_dev * dev)756f3da6577SLeon Romanovsky int mlx5_init_srq_table(struct mlx5_ib_dev *dev)
757f3da6577SLeon Romanovsky {
758f3da6577SLeon Romanovsky struct mlx5_srq_table *table = &dev->srq_table;
759f3da6577SLeon Romanovsky
760f3da6577SLeon Romanovsky memset(table, 0, sizeof(*table));
761b02a29ebSMatthew Wilcox xa_init_flags(&table->array, XA_FLAGS_LOCK_IRQ);
762f3da6577SLeon Romanovsky
763f3da6577SLeon Romanovsky table->nb.notifier_call = srq_event_notifier;
764f3da6577SLeon Romanovsky mlx5_notifier_register(dev->mdev, &table->nb);
765f3da6577SLeon Romanovsky
766f3da6577SLeon Romanovsky return 0;
767f3da6577SLeon Romanovsky }
768f3da6577SLeon Romanovsky
mlx5_cleanup_srq_table(struct mlx5_ib_dev * dev)769f3da6577SLeon Romanovsky void mlx5_cleanup_srq_table(struct mlx5_ib_dev *dev)
770f3da6577SLeon Romanovsky {
771f3da6577SLeon Romanovsky struct mlx5_srq_table *table = &dev->srq_table;
772f3da6577SLeon Romanovsky
773f3da6577SLeon Romanovsky mlx5_notifier_unregister(dev->mdev, &table->nb);
774f3da6577SLeon Romanovsky }
775