1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
21 	spin_lock_irqsave(&table->lock, flags);  in mlx5_get_rsc()
23 	common = radix_tree_lookup(&table->tree, rsn);  in mlx5_get_rsc()
25 		refcount_inc(&common->refcount);  in mlx5_get_rsc()
27 	spin_unlock_irqrestore(&table->lock, flags);  in mlx5_get_rsc()
34 	if (refcount_dec_and_test(&common->refcount))  in mlx5_core_put_rsc()
35 		complete(&common->free);  in mlx5_core_put_rsc()
97 	qpn = be32_to_cpu(eqe->data.dct.dctn) & 0xFFFFFF;  in dct_event_notifier()
98 	xa_lock_irqsave(&dev->qp_table.dct_xa, flags);  in dct_event_notifier()
99 	dct = xa_load(&dev->qp_table.dct_xa, qpn);  in dct_event_notifier()
101 		complete(&dct->drained);  in dct_event_notifier()
102 	xa_unlock_irqrestore(&dev->qp_table.dct_xa, flags);  in dct_event_notifier()
128 		rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;  in rsc_event_notifier()
129 		rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);  in rsc_event_notifier()
135 	common = mlx5_get_rsc(&dev->qp_table, rsn);  in rsc_event_notifier()
142 	switch (common->res) {  in rsc_event_notifier()
147 		qp->event(qp, event_type);  in rsc_event_notifier()
148 		/* Need to put resource in event handler */  in rsc_event_notifier()
162 	struct mlx5_qp_table *table = &dev->qp_table;  in create_resource_common()
165 	qp->common.res = rsc_type;  in create_resource_common()
166 	spin_lock_irq(&table->lock);  in create_resource_common()
167 	err = radix_tree_insert(&table->tree,  in create_resource_common()
168 				qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN),  in create_resource_common()
170 	spin_unlock_irq(&table->lock);  in create_resource_common()
174 	refcount_set(&qp->common.refcount, 1);  in create_resource_common()
175 	init_completion(&qp->common.free);  in create_resource_common()
176 	qp->pid = current->pid;  in create_resource_common()
184 	struct mlx5_qp_table *table = &dev->qp_table;  in destroy_resource_common()
187 	spin_lock_irqsave(&table->lock, flags);  in destroy_resource_common()
188 	radix_tree_delete(&table->tree,  in destroy_resource_common()
189 			  qp->qpn | (qp->common.res << MLX5_USER_INDEX_LEN));  in destroy_resource_common()
190 	spin_unlock_irqrestore(&table->lock, flags);  in destroy_resource_common()
192 	wait_for_completion(&qp->common.free);  in destroy_resource_common()
198 	u32 in[MLX5_ST_SZ_DW(destroy_dct_in)] = {};  in _mlx5_core_destroy_dct()  local
199 	struct mlx5_core_qp *qp = &dct->mqp;  in _mlx5_core_destroy_dct()
201 	MLX5_SET(destroy_dct_in, in, opcode, MLX5_CMD_OP_DESTROY_DCT);  in _mlx5_core_destroy_dct()
202 	MLX5_SET(destroy_dct_in, in, dctn, qp->qpn);  in _mlx5_core_destroy_dct()
203 	MLX5_SET(destroy_dct_in, in, uid, qp->uid);  in _mlx5_core_destroy_dct()
204 	return mlx5_cmd_exec_in(dev->mdev, destroy_dct, in);  in _mlx5_core_destroy_dct()
208 			 u32 *in, int inlen, u32 *out, int outlen)  in mlx5_core_create_dct()  argument
210 	struct mlx5_core_qp *qp = &dct->mqp;  in mlx5_core_create_dct()
213 	init_completion(&dct->drained);  in mlx5_core_create_dct()
214 	MLX5_SET(create_dct_in, in, opcode, MLX5_CMD_OP_CREATE_DCT);  in mlx5_core_create_dct()
216 	err = mlx5_cmd_do(dev->mdev, in, inlen, out, outlen);  in mlx5_core_create_dct()
220 	qp->qpn = MLX5_GET(create_dct_out, out, dctn);  in mlx5_core_create_dct()
221 	qp->uid = MLX5_GET(create_dct_in, in, uid);  in mlx5_core_create_dct()
222 	err = xa_err(xa_store_irq(&dev->qp_table.dct_xa, qp->qpn, dct, GFP_KERNEL));  in mlx5_core_create_dct()
233 		       u32 *in, int inlen, u32 *out)  in mlx5_qpc_create_qp()  argument
238 	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);  in mlx5_qpc_create_qp()
240 	err = mlx5_cmd_exec(dev->mdev, in, inlen, out,  in mlx5_qpc_create_qp()
245 	qp->uid = MLX5_GET(create_qp_in, in, uid);  in mlx5_qpc_create_qp()
246 	qp->qpn = MLX5_GET(create_qp_out, out, qpn);  in mlx5_qpc_create_qp()
252 	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)  in mlx5_qpc_create_qp()
253 		mlx5_debug_qp_add(dev->mdev, qp);  in mlx5_qpc_create_qp()
259 	MLX5_SET(destroy_qp_in, din, qpn, qp->qpn);  in mlx5_qpc_create_qp()
260 	MLX5_SET(destroy_qp_in, din, uid, qp->uid);  in mlx5_qpc_create_qp()
261 	mlx5_cmd_exec_in(dev->mdev, destroy_qp, din);  in mlx5_qpc_create_qp()
268 	u32 in[MLX5_ST_SZ_DW(drain_dct_in)] = {};  in mlx5_core_drain_dct()  local
269 	struct mlx5_core_qp *qp = &dct->mqp;  in mlx5_core_drain_dct()
271 	MLX5_SET(drain_dct_in, in, opcode, MLX5_CMD_OP_DRAIN_DCT);  in mlx5_core_drain_dct()
272 	MLX5_SET(drain_dct_in, in, dctn, qp->qpn);  in mlx5_core_drain_dct()
273 	MLX5_SET(drain_dct_in, in, uid, qp->uid);  in mlx5_core_drain_dct()
274 	return mlx5_cmd_exec_in(dev->mdev, drain_dct, in);  in mlx5_core_drain_dct()
280 	struct mlx5_qp_table *table = &dev->qp_table;  in mlx5_core_destroy_dct()
286 		if (dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)  in mlx5_core_destroy_dct()
291 	wait_for_completion(&dct->drained);  in mlx5_core_destroy_dct()
294 	tmp = xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, dct, XA_ZERO_ENTRY, GFP_KERNEL);  in mlx5_core_destroy_dct()
296 		return xa_err(tmp) ?: -EINVAL;  in mlx5_core_destroy_dct()
300 		xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, XA_ZERO_ENTRY, dct, 0);  in mlx5_core_destroy_dct()
303 	xa_erase_irq(&table->dct_xa, dct->mqp.qpn);  in mlx5_core_destroy_dct()
309 	u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};  in mlx5_core_destroy_qp()  local
311 	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)  in mlx5_core_destroy_qp()
312 		mlx5_debug_qp_remove(dev->mdev, qp);  in mlx5_core_destroy_qp()
316 	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);  in mlx5_core_destroy_qp()
317 	MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);  in mlx5_core_destroy_qp()
318 	MLX5_SET(destroy_qp_in, in, uid, qp->uid);  in mlx5_core_destroy_qp()
319 	return mlx5_cmd_exec_in(dev->mdev, destroy_qp, in);  in mlx5_core_destroy_qp()
325 	u32 in[MLX5_ST_SZ_DW(set_delay_drop_params_in)] = {};  in mlx5_core_set_delay_drop()  local
327 	MLX5_SET(set_delay_drop_params_in, in, opcode,  in mlx5_core_set_delay_drop()
329 	MLX5_SET(set_delay_drop_params_in, in, delay_drop_timeout,  in mlx5_core_set_delay_drop()
331 	return mlx5_cmd_exec_in(dev->mdev, set_delay_drop_params, in);  in mlx5_core_set_delay_drop()
335 	u32 *in;  member
343 	mbox->inlen  = inlen;  in mbox_alloc()
344 	mbox->outlen = outlen;  in mbox_alloc()
345 	mbox->in = kzalloc(mbox->inlen, GFP_KERNEL);  in mbox_alloc()
346 	mbox->out = kzalloc(mbox->outlen, GFP_KERNEL);  in mbox_alloc()
347 	if (!mbox->in || !mbox->out) {  in mbox_alloc()
348 		kfree(mbox->in);  in mbox_alloc()
349 		kfree(mbox->out);  in mbox_alloc()
350 		return -ENOMEM;  in mbox_alloc()
358 	kfree(mbox->in);  in mbox_free()
359 	kfree(mbox->out);  in mbox_free()
393 	mbox->out = NULL;  in modify_qp_mbox_alloc()
394 	mbox->in = NULL;  in modify_qp_mbox_alloc()
399 #define MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid)                            \  in modify_qp_mbox_alloc()  argument
401 		MLX5_SET(typ##_in, in, opcode, _opcode);                       \  in modify_qp_mbox_alloc()
402 		MLX5_SET(typ##_in, in, qpn, _qpn);                             \  in modify_qp_mbox_alloc()
403 		MLX5_SET(typ##_in, in, uid, _uid);                             \  in modify_qp_mbox_alloc()
406 #define MOD_QP_IN_SET_QPC(typ, in, _opcode, _qpn, _opt_p, _qpc, _uid)          \  in modify_qp_mbox_alloc()  argument
408 		MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid);                   \  in modify_qp_mbox_alloc()
409 		MLX5_SET(typ##_in, in, opt_param_mask, _opt_p);                \  in modify_qp_mbox_alloc()
410 		memcpy(MLX5_ADDR_OF(typ##_in, in, qpc), _qpc,                  \  in modify_qp_mbox_alloc()
418 			return -ENOMEM;  in modify_qp_mbox_alloc()
419 		MOD_QP_IN_SET(qp_2rst, mbox->in, opcode, qpn, uid);  in modify_qp_mbox_alloc()
423 			return -ENOMEM;  in modify_qp_mbox_alloc()
424 		MOD_QP_IN_SET(qp_2err, mbox->in, opcode, qpn, uid);  in modify_qp_mbox_alloc()
430 			return -ENOMEM;  in modify_qp_mbox_alloc()
431 		MOD_QP_IN_SET_QPC(rst2init_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
433 		MLX5_SET(rst2init_qp_in, mbox->in, ece, ece);  in modify_qp_mbox_alloc()
437 			return -ENOMEM;  in modify_qp_mbox_alloc()
438 		MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
440 		MLX5_SET(init2rtr_qp_in, mbox->in, ece, ece);  in modify_qp_mbox_alloc()
444 			return -ENOMEM;  in modify_qp_mbox_alloc()
445 		MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
447 		MLX5_SET(rtr2rts_qp_in, mbox->in, ece, ece);  in modify_qp_mbox_alloc()
451 			return -ENOMEM;  in modify_qp_mbox_alloc()
452 		MOD_QP_IN_SET_QPC(rts2rts_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
454 		MLX5_SET(rts2rts_qp_in, mbox->in, ece, ece);  in modify_qp_mbox_alloc()
458 			return -ENOMEM;  in modify_qp_mbox_alloc()
459 		MOD_QP_IN_SET_QPC(sqerr2rts_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
464 			return -ENOMEM;  in modify_qp_mbox_alloc()
465 		MOD_QP_IN_SET_QPC(sqd2rts_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
470 			return -ENOMEM;  in modify_qp_mbox_alloc()
471 		MOD_QP_IN_SET_QPC(init2init_qp, mbox->in, opcode, qpn,  in modify_qp_mbox_alloc()
473 		MLX5_SET(init2init_qp_in, mbox->in, ece, ece);  in modify_qp_mbox_alloc()
476 		return -EINVAL;  in modify_qp_mbox_alloc()
487 	err = modify_qp_mbox_alloc(dev->mdev, opcode, qp->qpn, opt_param_mask,  in mlx5_core_qp_modify()
488 				   qpc, &mbox, qp->uid, (ece) ? *ece : 0);  in mlx5_core_qp_modify()
492 	err = mlx5_cmd_exec(dev->mdev, mbox.in, mbox.inlen, mbox.out,  in mlx5_core_qp_modify()
504 	struct mlx5_qp_table *table = &dev->qp_table;  in mlx5_init_qp_table()
506 	spin_lock_init(&table->lock);  in mlx5_init_qp_table()
507 	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);  in mlx5_init_qp_table()
508 	xa_init(&table->dct_xa);  in mlx5_init_qp_table()
510 	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)  in mlx5_init_qp_table()
511 		mlx5_qp_debugfs_init(dev->mdev);  in mlx5_init_qp_table()
513 	table->nb.notifier_call = rsc_event_notifier;  in mlx5_init_qp_table()
514 	mlx5_notifier_register(dev->mdev, &table->nb);  in mlx5_init_qp_table()
521 	struct mlx5_qp_table *table = &dev->qp_table;  in mlx5_cleanup_qp_table()
523 	mlx5_notifier_unregister(dev->mdev, &table->nb);  in mlx5_cleanup_qp_table()
524 	if (dev->ib_dev.type != RDMA_DEVICE_TYPE_SMI)  in mlx5_cleanup_qp_table()
525 		mlx5_qp_debugfs_cleanup(dev->mdev);  in mlx5_cleanup_qp_table()
531 	u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {};  in mlx5_core_qp_query()  local
533 	MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);  in mlx5_core_qp_query()
534 	MLX5_SET(query_qp_in, in, qpn, qp->qpn);  in mlx5_core_qp_query()
535 	MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext);  in mlx5_core_qp_query()
537 	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, outlen);  in mlx5_core_qp_query()
543 	u32 in[MLX5_ST_SZ_DW(query_dct_in)] = {};  in mlx5_core_dct_query()  local
544 	struct mlx5_core_qp *qp = &dct->mqp;  in mlx5_core_dct_query()
546 	MLX5_SET(query_dct_in, in, opcode, MLX5_CMD_OP_QUERY_DCT);  in mlx5_core_dct_query()
547 	MLX5_SET(query_dct_in, in, dctn, qp->qpn);  in mlx5_core_dct_query()
549 	return mlx5_cmd_exec(dev->mdev, (void *)&in, sizeof(in), (void *)out,  in mlx5_core_dct_query()
556 	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)] = {};  in mlx5_core_xrcd_alloc()  local
559 	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);  in mlx5_core_xrcd_alloc()
560 	err = mlx5_cmd_exec_inout(dev->mdev, alloc_xrcd, in, out);  in mlx5_core_xrcd_alloc()
568 	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)] = {};  in mlx5_core_xrcd_dealloc()  local
570 	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);  in mlx5_core_xrcd_dealloc()
571 	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);  in mlx5_core_xrcd_dealloc()
572 	return mlx5_cmd_exec_in(dev->mdev, dealloc_xrcd, in);  in mlx5_core_xrcd_dealloc()
577 	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};  in destroy_rq_tracked()  local
579 	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);  in destroy_rq_tracked()
580 	MLX5_SET(destroy_rq_in, in, rqn, rqn);  in destroy_rq_tracked()
581 	MLX5_SET(destroy_rq_in, in, uid, uid);  in destroy_rq_tracked()
582 	return mlx5_cmd_exec_in(dev->mdev, destroy_rq, in);  in destroy_rq_tracked()
585 int mlx5_core_create_rq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,  in mlx5_core_create_rq_tracked()  argument
591 	err = mlx5_core_create_rq(dev->mdev, in, inlen, &rqn);  in mlx5_core_create_rq_tracked()
595 	rq->uid = MLX5_GET(create_rq_in, in, uid);  in mlx5_core_create_rq_tracked()
596 	rq->qpn = rqn;  in mlx5_core_create_rq_tracked()
604 	destroy_rq_tracked(dev, rq->qpn, rq->uid);  in mlx5_core_create_rq_tracked()
613 	return destroy_rq_tracked(dev, rq->qpn, rq->uid);  in mlx5_core_destroy_rq_tracked()
618 	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};  in destroy_sq_tracked()  local
620 	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);  in destroy_sq_tracked()
621 	MLX5_SET(destroy_sq_in, in, sqn, sqn);  in destroy_sq_tracked()
622 	MLX5_SET(destroy_sq_in, in, uid, uid);  in destroy_sq_tracked()
623 	mlx5_cmd_exec_in(dev->mdev, destroy_sq, in);  in destroy_sq_tracked()
626 int mlx5_core_create_sq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,  in mlx5_core_create_sq_tracked()  argument
632 	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);  in mlx5_core_create_sq_tracked()
633 	err = mlx5_cmd_exec(dev->mdev, in, inlen, out, sizeof(out));  in mlx5_core_create_sq_tracked()
637 	sq->qpn = MLX5_GET(create_sq_out, out, sqn);  in mlx5_core_create_sq_tracked()
638 	sq->uid = MLX5_GET(create_sq_in, in, uid);  in mlx5_core_create_sq_tracked()
646 	destroy_sq_tracked(dev, sq->qpn, sq->uid);  in mlx5_core_create_sq_tracked()
655 	destroy_sq_tracked(dev, sq->qpn, sq->uid);  in mlx5_core_destroy_sq_tracked()
663 	struct mlx5_qp_table *table = &dev->qp_table;  in mlx5_core_res_hold()