684 static void destroy_user_rq(struct ib_pd *pd, struct mlx5_ib_rwq *rwq,  in destroy_user_rq()  argument
693 	mlx5_ib_db_unmap_user(context, &rwq->db);  in destroy_user_rq()
694 	if (rwq->umem)  in destroy_user_rq()
695 		ib_umem_release(rwq->umem);  in destroy_user_rq()
699 			  struct mlx5_ib_rwq *rwq,  in create_user_rq()  argument
713 	rwq->umem = ib_umem_get(pd->uobject->context, ucmd->buf_addr,  in create_user_rq()
714 			       rwq->buf_size, 0, 0);  in create_user_rq()
715 	if (IS_ERR(rwq->umem)) {  in create_user_rq()
717 		err = PTR_ERR(rwq->umem);  in create_user_rq()
721 	mlx5_ib_cont_pages(rwq->umem, ucmd->buf_addr, 0, &npages, &page_shift,  in create_user_rq()
724 				     &rwq->rq_page_offset);  in create_user_rq()
730 	rwq->rq_num_pas = ncont;  in create_user_rq()
731 	rwq->page_shift = page_shift;  in create_user_rq()
732 	rwq->log_page_size =  page_shift - MLX5_ADAPTER_PAGE_SHIFT;  in create_user_rq()
733 	rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);  in create_user_rq()
736 		    (unsigned long long)ucmd->buf_addr, rwq->buf_size,  in create_user_rq()
739 	err = mlx5_ib_db_map_user(context, ucmd->db_addr, &rwq->db);  in create_user_rq()
745 	rwq->create_type = MLX5_WQ_USER;  in create_user_rq()
749 	ib_umem_release(rwq->umem);  in create_user_rq()
4670 	struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);  in mlx5_ib_wq_event()  local
4671 	struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);  in mlx5_ib_wq_event()
4674 	if (rwq->ibwq.event_handler) {  in mlx5_ib_wq_event()
4675 		event.device     = rwq->ibwq.device;  in mlx5_ib_wq_event()
4676 		event.element.wq = &rwq->ibwq;  in mlx5_ib_wq_event()
4686 		rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);  in mlx5_ib_wq_event()
4690 static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,  in create_rq()  argument
4703 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;  in create_rq()
4712 	MLX5_SET(rqc, rqc, user_index, rwq->user_index);  in create_rq()
4719 	MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);  in create_rq()
4720 	MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);  in create_rq()
4722 	MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);  in create_rq()
4723 	MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);  in create_rq()
4724 	MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);  in create_rq()
4725 	MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);  in create_rq()
4727 	mlx5_ib_populate_pas(dev, rwq->umem, rwq->page_shift, rq_pas0, 0);  in create_rq()
4728 	err = mlx5_core_create_rq_tracked(dev->mdev, in, inlen, &rwq->core_qp);  in create_rq()
4736 			    struct mlx5_ib_rwq *rwq)  in set_user_rq_size()  argument
4745 	rwq->wqe_count = ucmd->rq_wqe_count;  in set_user_rq_size()
4746 	rwq->wqe_shift = ucmd->rq_wqe_shift;  in set_user_rq_size()
4747 	rwq->buf_size = (rwq->wqe_count << rwq->wqe_shift);  in set_user_rq_size()
4748 	rwq->log_rq_stride = rwq->wqe_shift;  in set_user_rq_size()
4749 	rwq->log_rq_size = ilog2(rwq->wqe_count);  in set_user_rq_size()
4756 			   struct mlx5_ib_rwq *rwq)  in prepare_user_rq()  argument
4791 	err = set_user_rq_size(dev, init_attr, &ucmd, rwq);  in prepare_user_rq()
4797 	err = create_user_rq(dev, pd, rwq, &ucmd);  in prepare_user_rq()
4804 	rwq->user_index = ucmd.user_index;  in prepare_user_rq()
4813 	struct mlx5_ib_rwq *rwq;  in mlx5_ib_create_wq()  local
4828 		rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);  in mlx5_ib_create_wq()
4829 		if (!rwq)  in mlx5_ib_create_wq()
4831 		err = prepare_user_rq(pd, init_attr, udata, rwq);  in mlx5_ib_create_wq()
4834 		err = create_rq(rwq, pd, init_attr);  in mlx5_ib_create_wq()
4844 	rwq->ibwq.wq_num = rwq->core_qp.qpn;  in mlx5_ib_create_wq()
4845 	rwq->ibwq.state = IB_WQS_RESET;  in mlx5_ib_create_wq()
4854 	rwq->core_qp.event = mlx5_ib_wq_event;  in mlx5_ib_create_wq()
4855 	rwq->ibwq.event_handler = init_attr->event_handler;  in mlx5_ib_create_wq()
4856 	return &rwq->ibwq;  in mlx5_ib_create_wq()
4859 	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);  in mlx5_ib_create_wq()
4861 	destroy_user_rq(pd, rwq, udata);  in mlx5_ib_create_wq()
4863 	kfree(rwq);  in mlx5_ib_create_wq()
4870 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);  in mlx5_ib_destroy_wq()  local
4872 	mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp);  in mlx5_ib_destroy_wq()
4873 	destroy_user_rq(wq->pd, rwq, udata);  in mlx5_ib_destroy_wq()
4874 	kfree(rwq);  in mlx5_ib_destroy_wq()
4970 	struct mlx5_ib_rwq *rwq = to_mrwq(wq);  in mlx5_ib_modify_wq()  local
5002 	MLX5_SET(modify_rq_in, in, rqn, rwq->core_qp.qpn);  in mlx5_ib_modify_wq()
5018 		rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;  in mlx5_ib_modify_wq()